diff --git "a/hypernet/scale_100repos_results/cr_test/results.json" "b/hypernet/scale_100repos_results/cr_test/results.json"
new file mode 100644--- /dev/null
+++ "b/hypernet/scale_100repos_results/cr_test/results.json"
@@ -0,0 +1,57736 @@
+{
+  "exact_match_pct": 61.27221702525725,
+  "exact_match_count": 3930,
+  "n": 6414,
+  "n_total": 6414,
+  "code_bleu": 0.7535512690205965,
+  "edit_similarity": 0.755810624800875,
+  "entries": [
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "real_repr",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.15384615384615385
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "None",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "stderr",
+      "got": "stderr",
+      "got_raw": "stderr",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "report",
+      "got": "report",
+      "got_raw": "report",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "returncode",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "f\"ERROR: {e}\\n\"",
+      "got": "str(e)",
+      "got_raw": "str(e)",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.09523809523809523
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "reported_categories",
+      "got": "reported_categories",
+      "got_raw": "reported_categories",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "self._changed_files(tmp_path)",
+      "got": "self._changed_files(tmp_path)",
+      "got_raw": "self._changed_files(tmp_path)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "normalize(report_output.getvalue())",
+      "got": "session.report",
+      "got_raw": "session.report",
+      "exact_match": false,
+      "code_bleu": 0.1694253756768459,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "parse_outcomes(result_stdout.splitlines())",
+      "got": "parse_outcomes(result_stdout.splitlines())",
+      "got_raw": "parse_outcomes(result_stdout.splitlines())",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "num_args",
+      "got": "num_args",
+      "got_raw": "num_args",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "number",
+      "got": "number",
+      "got_raw": "number",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "self.header",
+      "got": "self.header",
+      "got_raw": "self.header",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "reported_flags",
+      "got": "flags",
+      "got_raw": "flags",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5263157894736842
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "\"fix\" in s.flags",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"mxxm\")",
+      "got": "snapshot(\"mddiimx\")",
+      "got_raw": "snapshot(\"mddiimx\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"dmmmi\")",
+      "got": "(\"iabc\"",
+      "got_raw": "(\"iabc\", \"abcd\")",
+      "exact_match": false,
+      "code_bleu": 0.2686424829558855,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"mddiim\")",
+      "got": "\"axyc\"",
+      "got_raw": "\"axyc\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "new_code",
+      "got": "new_code",
+      "got_raw": "new_code",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "eval(code)",
+      "got": "eval(code)",
+      "got_raw": "eval(code)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"B()\")",
+      "got": "\"0\"",
+      "got_raw": "\"0\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "HasRepr(int, \"5\")",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.1111111111111111
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"B(b=5)\")",
+      "got": "snapshot(\"B(b=5)\")",
+      "got_raw": "snapshot(\"B(b=5)\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"C(a=1, c=2)\")",
+      "got": "\"DataClass(a=1, c=2)\"",
+      "got_raw": "\"DataClass(a=1, c=2)\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.5909090909090909
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"FakeTuple()\")",
+      "got": "\"FakeTuple()\"",
+      "got_raw": "\"FakeTuple()\"",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.7222222222222222
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"Dataclass(a=0)\")",
+      "got": "snapshot(\"Dataclass(a=0",
+      "got_raw": "snapshot(\"Dataclass(a=0, b=0, c=[])\")",
+      "exact_match": false,
+      "code_bleu": 0.21105340631872638,
+      "edit_similarity": 0.9387755102040817
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"{'a', 'b', 1, 2, 3}\")",
+      "got": "\"{1, 2, 3, 'a', True, 'b'}\"",
+      "got_raw": "\"{1, 2, 3, 'a', True, 'b'}\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.3793103448275862
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"Counter({1: 3, 2: 1})\")",
+      "got": "snapshot(\"Counter([1, 1, 1, 2])\")",
+      "got_raw": "snapshot(\"Counter([1, 1, 1, 2])\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.7878787878787878
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"{1, 2, 3, 10, 11, 20, 200}\" )",
+      "got": "snapshot(\"{1, 2, 3, 10, 11, 20, 200}\")",
+      "got_raw": "snapshot(\"{1, 2, 3, 10, 11, 20, 200}\")",
+      "exact_match": false,
+      "code_bleu": 1.0,
+      "edit_similarity": 0.9743589743589743
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"{'a', 'b', 1j, 2j, 3j, True}\" )",
+      "got": "snapshot(\"{'a', 'b', 1j, 2j, 3j}\")",
+      "got_raw": "snapshot(\"{'a', 'b', 1j, 2j, 3j}\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.8947368421052632
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ class Color: pass assert [Color,int] == snapshot([Color, int]) \"\"\" )",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.04597701149425287
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ from enum import Flag, auto class Color(Flag): red = auto() green = auto() blue = auto() assert Color.red | Color.blue == snapshot(Color.red | Color.blue) \"\"\" )",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.0335195530726257
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ from enum import Enum from dataclasses import dataclass class color(Enum): red=\"red\" blue=\"blue\" @dataclass class container: bg: color=color.red fg: color=color.blue assert container(bg=color.red,fg=color.red) == snapshot(container(fg=color.red)) \"\"\" )",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.05,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "blocks",
+      "got": "blocks",
+      "got_raw": "blocks",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "new_code",
+      "got": "file",
+      "got_raw": "file",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "recorded_markdown_code",
+      "got": "recorded_markdown_code",
+      "got_raw": "recorded_markdown_code",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "[0, 5]",
+      "got": "[0, \"ef2d127de37b000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+      "got_raw": "[0, \"ef2d127de37b000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.058394160583941604
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"5\")",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.14285714285714285
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"[0, 5]\")",
+      "got": "\"5\"",
+      "got_raw": "\"5\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "A(a=outsource(5), b=2)",
+      "got": "s",
+      "got_raw": "s",
+      "exact_match": false,
+      "code_bleu": 0.038461538461538464,
+      "edit_similarity": 0.08695652173913043
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"A(a=5, b=2)\")",
+      "got": "\"{'a': 5, 'b': 2}\"",
+      "got_raw": "\"{'a': 5, 'b': 2}\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.3902439024390244
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "HasRepr(float, \"nan\")",
+      "got": "float(\"nan\")",
+      "got_raw": "float(\"nan\")",
+      "exact_match": false,
+      "code_bleu": 0.35818997727451396,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "HasRepr(Thing, \"<something>\")",
+      "got": "HasRepr(Thing)",
+      "got_raw": "HasRepr(Thing)",
+      "exact_match": false,
+      "code_bleu": 0.4714045207910316,
+      "edit_similarity": 0.6511627906976745
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "code",
+      "got": "code",
+      "got_raw": "code",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "set()",
+      "got": "all_flags",
+      "got_raw": "all_flags",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.14285714285714285
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "test2",
+      "got": "test2",
+      "got_raw": "test2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "s.error",
+      "got": "s.is_fixed",
+      "got_raw": "s.is_fixed",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.35294117647058826
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "s.source",
+      "got": "code",
+      "got_raw": "code",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "s2.source",
+      "got": "s2.source",
+      "got_raw": "s2.source",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot()",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "TypeError)",
+      "got": "TypeError)",
+      "got_raw": "TypeError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(1)",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"\")",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "\"snapshot()\\n\"",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "AssertionError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5384615384615384
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "{reported_flag}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(set())",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.15060033393497413,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "str(error.value)",
+      "got": "str(error.value)",
+      "got_raw": "str(error.value)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "expected_warnings",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "\"Flags({'update'})\"",
+      "got": "\"Flags({'update'})\"",
+      "got_raw": "\"Flags({'update'})\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot({\"create\"})",
+      "got": "Flags()",
+      "got_raw": "Flags()",
+      "exact_match": false,
+      "code_bleu": 0.24999999999999997,
+      "edit_similarity": 0.2962962962962963
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "repr(Is(5))",
+      "got": "repr(5)",
+      "got_raw": "repr(5)",
+      "exact_match": false,
+      "code_bleu": 0.43419182037233855,
+      "edit_similarity": 0.7777777777777778
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "flags",
+      "got": "all_flags",
+      "got_raw": "all_flags",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7142857142857143
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "s1.error",
+      "got": "(\"fix\"",
+      "got_raw": "(\"fix\" in s1.flags)",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "new_code",
+      "got": "new_code",
+      "got_raw": "new_code",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"assert {}==snapshot({})\")",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"assert [1,2]==snapshot([0+1,2])\")",
+      "got": "[1, 2, 3]",
+      "got_raw": "[1, 2, 3]",
+      "exact_match": false,
+      "code_bleu": 0.1561969968460128,
+      "edit_similarity": 0.23076923076923078
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"assert {1:1}==snapshot({1:0+1})\")",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"assert [1,5]==snapshot([0+1, 5])\")",
+      "got": "[1, 2, 3]",
+      "got_raw": "[1, 2, 3]",
+      "exact_match": false,
+      "code_bleu": 0.1561969968460128,
+      "edit_similarity": 0.22641509433962265
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"assert (1,5)==snapshot((0+1, 5))\")",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot('assert {1+2:\"3\"}==snapshot({1+2:\"3\"})')",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"assert {1:1, 2:2}==snapshot({1:0+1, 2:2})\")",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"assert [1,2,3,4,5,6]==snapshot([0+1, 2, 3, 4, 5, 6])\")",
+      "got": "[1, 2, 3]",
+      "got_raw": "[1, 2, 3]",
+      "exact_match": false,
+      "code_bleu": 0.1561969968460128,
+      "edit_similarity": 0.2465753424657534
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( 'assert {0:\"before\",1:1,2:\"after\"}==snapshot({0: \"before\", 1:0+1, 2: \"after\"})' )",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ from dirty_equals import IsInt assert {5:5,2:2}==snapshot({5:IsInt(),2:2}) \"\"\" )",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.06060606060606061
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ left = { \"a\": 1, \"b\": { \"c\": 2, \"d\": [ 3, 4, 5, ], }, \"e\": ( { \"f\": 6, \"g\": 7, }, ), } assert left == snapshot( { \"a\": 1, \"b\": { \"c\": 2 * 1 + 0, \"d\": [ int(3), 4, 5, ], }, \"e\": ({\"f\": 3 + 3, \"g\": 7},), } ) \"\"\" )",
+      "got": "\"foo\"",
+      "got_raw": "\"foo\"",
+      "exact_match": false,
+      "code_bleu": 0.014705882352941176,
+      "edit_similarity": 0.017316017316017316
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"\")",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.2259005009024612,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ def test_a(): assert 5==snapshot(5) \"\"\" )",
+      "got": "\"\"\"\\",
+      "got_raw": "\"\"\"\\\ndef test_a():\n    assert 5==snapshot(5)\"\"\"",
+      "exact_match": false,
+      "code_bleu": 0.2259005009024612,
+      "edit_similarity": 0.13333333333333333
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ def test_a(): assert 4==snapshot(4) \"\"\" )",
+      "got": "\"\"\"\\",
+      "got_raw": "\"\"\"\\\ndef test_a():\n    assert 4==5\"\"\"",
+      "exact_match": false,
+      "code_bleu": 0.2259005009024612,
+      "edit_similarity": 0.13333333333333333
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ def test_a(): assert 5 in snapshot([5]) \"\"\" )",
+      "got": "\"\"\"\\",
+      "got_raw": "\"\"\"\\\ndef test_a():\n    assert 5 in snapshot([5])\"\"\"",
+      "exact_match": false,
+      "code_bleu": 0.2259005009024612,
+      "edit_similarity": 0.125
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ E assert 2 == 1 E + where 1 = snapshot() \"\"\" )",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.15060033393497413,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ def test_a(): assert \"5\" == snapshot(\"5\") \"\"\" )",
+      "got": "\"\"\"\\",
+      "got_raw": "\"\"\"\\\ndef test_a():\n    assert \"5\" == snapshot(\"5\")\"\"\"",
+      "exact_match": false,
+      "code_bleu": 0.1355403005414767,
+      "edit_similarity": 0.12121212121212122
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ E assert 2 == 1 E + where 1 = snapshot(1) \"\"\" )",
+      "got": "[1]",
+      "got_raw": "[1]",
+      "exact_match": false,
+      "code_bleu": 0.18995892141289814,
+      "edit_similarity": 0.03076923076923077
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ def test_something(): assert 2 == snapshot(2) \"\"\" )",
+      "got": "\"\"\"\\",
+      "got_raw": "\"\"\"\\\ndef test_something():\n    assert 2 == snapshot(2)\"\"\"",
+      "exact_match": false,
+      "code_bleu": 0.2259005009024612,
+      "edit_similarity": 0.11428571428571428
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"ERROR: --inline-snapshot=disable can not be combined with other flags (fix)\" )",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.04395604395604396
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"ERROR: --inline-snapshot=disable can not be combined with other flags (review)\" )",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.0425531914893617
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ > assert \"a\"==snapshot(\"b\") E AssertionError: assert 'a' == 'b' E \\n\\ E - b E + a \"\"\" )",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.09036020036098448,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ > assert snapshot(\"b\")==\"a\" E AssertionError: assert 'b' == 'a' E \\n\\ E - a E + b \"\"\" )",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.09036020036098448,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ def test_a(): assert list(range(10)) == snapshot( [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] ) \"\"\" )",
+      "got": "\"\"\"\\",
+      "got_raw": "\"\"\"\\\ndef test_a():\n    assert list(range(10)) == snapshot([])",
+      "exact_match": false,
+      "code_bleu": 0.2259005009024612,
+      "edit_similarity": 0.07476635514018691
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ def test_a(): assert \"5\" == snapshot('''5''') assert 5 <= snapshot(5) assert 5 == snapshot(5) \"\"\" )",
+      "got": "\"\"\"\\",
+      "got_raw": "\"\"\"\\\ndef test_a():\n    assert \"5\" == snapshot('''5''')",
+      "exact_match": false,
+      "code_bleu": 0.1694253756768459,
+      "edit_similarity": 0.06779661016949153
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ def test_a(): assert \"5\" == snapshot('''5''') assert 5 <= snapshot(5) assert 5 == snapshot(4) \"\"\" )",
+      "got": "\"\"\"\\",
+      "got_raw": "\"\"\"\\\ndef test_a():\n    assert \"5\" == snapshot(\"5\")",
+      "exact_match": false,
+      "code_bleu": 0.1694253756768459,
+      "edit_similarity": 0.06779661016949153
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "b",
+      "got": "b",
+      "got_raw": "b",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "a",
+      "got": "a",
+      "got_raw": "a",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "r",
+      "got": "(1",
+      "got_raw": "(1, 2, 2, 5)",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "\"\"\" 12a45 1245 12c345 \"\"\"",
+      "got": "b\"a\\n12345\\n12345\\n\"",
+      "got_raw": "b\"a\\n12345\\n12345\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.3111111111111111
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "s",
+      "got": "s",
+      "got_raw": "s",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"s=snapshot('\\\\n')\")",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.12903225806451613
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot('s=snapshot(\"a\\\\n\")')",
+      "got": "\"abc\"",
+      "got_raw": "\"abc\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.17142857142857143
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"s=snapshot('abc\\\\n')\")",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.11764705882352941
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "'assert \"ab\" == snapshot(\"a\" \"b\")'",
+      "got": "\"foo\"",
+      "got_raw": "\"foo\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.15384615384615385
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot('s = snapshot(b\"a\\\\nb\")')",
+      "got": "s",
+      "got_raw": "s",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.05714285714285714
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "'assert \"ab\" == snapshot(\"a\"\\n \"b\")'",
+      "got": "\"update\"",
+      "got_raw": "\"update\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.18181818181818182
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "'assert b\"ab\" == snapshot(b\"a\"\\n b\"b\")'",
+      "got": "'assert \"ab\" == snapshot(\"a\" \"b\")'",
+      "got_raw": "'assert \"ab\" == snapshot(\"a\" \"b\")'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9315068493150684
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( '''\\ s=snapshot(\"\"\"\\\\ a a \"\"\")\\ ''' )",
+      "got": "snapshot(\"\"\"\\",
+      "got_raw": "snapshot(\"\"\"\\\ns=snapshot(\"\"\"\\\na\na\\\\\n\"\"\")\\\n\"\"\")",
+      "exact_match": false,
+      "code_bleu": 0.29942009330257024,
+      "edit_similarity": 0.43333333333333335
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( '''\\ s = snapshot(\"\"\"\\\\ '\\\\ \"\"\")\\ ''' )",
+      "got": "snapshot('s = snapshot(\"\\\\\\\\n\\\\\\'\")')",
+      "got_raw": "snapshot('s = snapshot(\"\\\\\\\\n\\\\\\'\")')",
+      "exact_match": false,
+      "code_bleu": 0.15634853789372763,
+      "edit_similarity": 0.7441860465116279
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( '''\\ s = snapshot(\"\"\"\\\\ \"\\\\ \"\"\")\\ ''' )",
+      "got": "snapshot('s = snapshot(\"\\\\\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\",
+      "got_raw": "snapshot('s = snapshot(\"\\\\\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\",
+      "exact_match": false,
+      "code_bleu": 0.008302982018071978,
+      "edit_similarity": 0.14578587699316628
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( '''\\ s = snapshot(\"\"\"\\\\ a b\\\\ \"\"\")\\ ''' )",
+      "got": "'s = snapshot(\"a\\nb\")'",
+      "got_raw": "'s = snapshot(\"a\\nb\")'",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.547945205479452
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( '''\\ s = snapshot(\"\"\"\\\\ \\\\x08\\\\ \"\"\")\\ ''' )",
+      "got": "snapshot('s = snapshot(\"\\\\n\\\\b\")')",
+      "got_raw": "snapshot('s = snapshot(\"\\\\n\\\\b\")')",
+      "exact_match": false,
+      "code_bleu": 0.3387551654364098,
+      "edit_similarity": 0.735632183908046
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( '''\\ assert \"ab\\\\nc\" == snapshot(\"\"\"\\\\ ab c\\\\ \"\"\")\\ ''' )",
+      "got": "\"ab\"",
+      "got_raw": "\"ab\"",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.11267605633802817
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "4",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"\")",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot([])",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.28284271247461906,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"ERROR: --inline-snapshot=create can not be combined with xdist\" )",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.05128205128205128
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ Error: one snapshot has incorrect values (--inline-snapshot=fix) You can also use --inline-snapshot=review to approve the changes interactively \"\"\" )",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.09910059291689341,
+      "edit_similarity": 0.024096385542168676
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot(\"\")",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot([])",
+      "got": "{",
+      "got_raw": "{ \"test\": \"test\" }",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "project.source",
+      "got": "project.source",
+      "got_raw": "project.source",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "outsource(\"one\")",
+      "got": "\"one\"",
+      "got_raw": "\"one\"",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.47619047619047616
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "outsource(\"two\")",
+      "got": "outsource(\"two\")",
+      "got_raw": "outsource(\"two\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "external(\"hash:bbbbb*.txt\")",
+      "got": "\"test4\"",
+      "got_raw": "\"test4\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.29411764705882354
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ from os import environ",
+      "got": "b\"from os import chdir\\n\"",
+      "got_raw": "b\"from os import chdir\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.10040022262331608,
+      "edit_similarity": 0.6129032258064516
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( [\"9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08.txt\"] )",
+      "got": "snapshot([])",
+      "got_raw": "snapshot([])",
+      "exact_match": false,
+      "code_bleu": 0.503562566226128,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( [\"2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae.txt\"] )",
+      "got": "{ \"foo\": \"foo\"",
+      "got_raw": "{ \"foo\": \"foo\", \"blub\": \"blub\", }",
+      "exact_match": false,
+      "code_bleu": 0.2014250264904512,
+      "edit_similarity": 0.10204081632653061
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ from os import environ from os import getcwd from os import chdir \"\"\" )",
+      "got": "\"\"\"\\",
+      "got_raw": "\"\"\"\\\nfrom os import environ, chdir\nfrom os import getcwd",
+      "exact_match": false,
+      "code_bleu": 0.2259005009024612,
+      "edit_similarity": 0.08888888888888889
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ ''' docstring ''' from __future__ import annotations from os import chdir \"\"\" )",
+      "got": "\"\"\"\\",
+      "got_raw": "\"\"\"\\\nfrom __future__ import annotations",
+      "exact_match": false,
+      "code_bleu": 0.2259005009024612,
+      "edit_similarity": 0.08163265306122448
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"assert outsource('text') == snapshot(external(\\\"uuid:f728b4fa-4248-4e3a-8a5d-2f346baa9455.txt\\\"))\" )",
+      "got": "\"1\"",
+      "got_raw": "\"1\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.03508771929824561
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( [ ExternalLocation( storage=\"hash\", stem=\"111*\", suffix=\".txt\", filename=Path(\"a.py\"), linenumber=1, ) ] )",
+      "got": "True(True)",
+      "got_raw": "True(True)",
+      "exact_match": false,
+      "code_bleu": 0.05130778895271294,
+      "edit_similarity": 0.09523809523809523
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ > assert outsource(\"test2\") == s E AssertionError: assert 'test2' == 'test' E \\n\\ E - test E + test2 E ? + \"\"\" )",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.1129502504512306,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( \"\"\"\\ from inline_snapshot import external def test_a(): assert outsource(\"test\") == snapshot(external(\"hash:9f86d081884c*.txt\")) \"\"\" )",
+      "got": "default_result",
+      "got_raw": "default_result",
+      "exact_match": false,
+      "code_bleu": 0.03125,
+      "edit_similarity": 0.12658227848101267
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "snapshot( [ \"60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752.txt\", \"9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08.txt\", ] )",
+      "got": "{ \"test\": \"test\", \"test2\": \"test2\", }",
+      "got_raw": "{ \"test\": \"test\", \"test2\": \"test2\", }",
+      "exact_match": false,
+      "code_bleu": 0.13950796967929133,
+      "edit_similarity": 0.1326530612244898
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "\"test\"",
+      "got": "\"test\"",
+      "got_raw": "\"test\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "results[name]",
+      "got": "results[name]",
+      "got_raw": "results[name]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "15r10nk/inline-snapshot",
+      "expected": "f\"{type(e).__name__}: {e}\"",
+      "got": "e",
+      "got_raw": "e",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.07407407407407407
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "3",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "name2",
+      "got": "name2",
+      "got_raw": "name2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "name1",
+      "got": "name1",
+      "got_raw": "name1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "LANG_TO_INDEX",
+      "got": "LANG_TO_INDEX",
+      "got_raw": "LANG_TO_INDEX",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "skip",
+      "got": "skip",
+      "got_raw": "skip",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "False",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "source",
+      "got": "source",
+      "got_raw": "source",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "\"reference\"",
+      "got": "\"reference\"",
+      "got_raw": "\"reference\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "field_names",
+      "got": "field_names",
+      "got_raw": "field_names",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "\"persona_no_ge\"",
+      "got": "\"persona_no_ge\"",
+      "got_raw": "\"persona_no_ge\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "\"persona_with_ge\"",
+      "got": "\"persona_with_ge\"",
+      "got_raw": "\"persona_with_ge\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "ctx.get_skip_components()",
+      "got": "ctx",
+      "got_raw": "ctx",
+      "exact_match": false,
+      "code_bleu": 0.11892071150027211,
+      "edit_similarity": 0.21428571428571427
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "\"ja\"",
+      "got": "\"ja\"",
+      "got_raw": "\"ja\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "\"en\"",
+      "got": "\"en\"",
+      "got_raw": "\"en\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "\"zh\"",
+      "got": "\"zh\"",
+      "got_raw": "\"zh\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "config",
+      "got": "config",
+      "got_raw": "config",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "'new.wav'",
+      "got": "'new.wav'",
+      "got_raw": "'new.wav'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "ext.lower()",
+      "got": "ext.lower()",
+      "got_raw": "ext.lower()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "SUPPORTED_AUDIO_EXTS",
+      "got": "SUPPORTED_AUDIO_EXTS",
+      "got_raw": "SUPPORTED_AUDIO_EXTS",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "\"gpu\"",
+      "got": "\"gpu\"",
+      "got_raw": "\"gpu\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "EnvironmentMismatchError)",
+      "got": "EnvironmentMismatchError)",
+      "got_raw": "EnvironmentMismatchError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "EnvironmentStatus.CPU_ONLY",
+      "got": "EnvironmentStatus.CPU_ONLY",
+      "got_raw": "EnvironmentStatus.CPU_ONLY",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "EnvironmentStatus.GPU_READY",
+      "got": "EnvironmentStatus.GPU_READY",
+      "got_raw": "EnvironmentStatus.GPU_READY",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "EnvironmentStatus.GPU_DEPS_MISSING",
+      "got": "EnvironmentStatus.DEPENDENCIES_MISSING",
+      "got_raw": "EnvironmentStatus.DEPENDENCIES_MISSING",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.8333333333333334
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "frontend2",
+      "got": "frontend2",
+      "got_raw": "frontend2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "frontend3",
+      "got": "frontend3",
+      "got_raw": "frontend3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "languages",
+      "got": "languages",
+      "got_raw": "languages",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "mock_frontend",
+      "got": "mock_frontend",
+      "got_raw": "mock_frontend",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "mock_instance_2",
+      "got": "mock_instance_2",
+      "got_raw": "mock_instance_2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "ValueError, match=\"Unsupported language\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.4230769230769231
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "mgr2",
+      "got": "mgr2",
+      "got_raw": "mgr2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "status",
+      "got": "status",
+      "got_raw": "status",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "\"v2\"",
+      "got": "\"2.0.0\"",
+      "got_raw": "\"2.0.0\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5454545454545454
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "inputs",
+      "got": "inputs",
+      "got_raw": "inputs",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "\"v2Pro\"",
+      "got": "\"v2-pro\"",
+      "got_raw": "\"v2-pro\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "V2_SPEC",
+      "got": "V2_SPEC",
+      "got_raw": "V2_SPEC",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "\"v2ProPlus\"",
+      "got": "\"v2ProPlus\"",
+      "got_raw": "\"v2ProPlus\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "V2_PRO_SPEC",
+      "got": "V2_PRO_SPEC",
+      "got_raw": "V2_PRO_SPEC",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "[\"a\", \"b\", \"c\"]",
+      "got": "[\"a\", \"b\", \"c\"]",
+      "got_raw": "[\"a\", \"b\", \"c\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "V2_PRO_PLUS_SPEC",
+      "got": "V2_PRO_PLUS_SPEC",
+      "got_raw": "V2_PRO_PLUS_SPEC",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "source",
+      "got": "source",
+      "got_raw": "source",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "skip_components",
+      "got": "skip_components",
+      "got_raw": "skip_components",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "first_call_count",
+      "got": "first_call_count",
+      "got_raw": "first_call_count",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "Lux-Luna/LunaVox",
+      "expected": "kwargs.get(\"skip_components\", set())",
+      "got": "kwargs",
+      "got_raw": "kwargs",
+      "exact_match": false,
+      "code_bleu": 0.05946035575013606,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "content",
+      "got": "content",
+      "got_raw": "content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "output_path / \"index.html\")",
+      "got": "output_path / \"slides.html\")",
+      "got_raw": "output_path / \"slides.html\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "output_path / \"randomfile-1.txt\")",
+      "got": "output_path / \"someslides-3.html\")",
+      "got_raw": "output_path / \"someslides-3.html\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.6865671641791045
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "output_path / \"someslides-1.html\")",
+      "got": "output_path / \"slides.html\")",
+      "got_raw": "output_path / \"slides.html\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.9032258064516129
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "output_path / \"extra/randomfile-3.txt\")",
+      "got": "output_path / \"somefolder/somefile-1.txt\")",
+      "got_raw": "output_path / \"somefolder/somefile-1.txt\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.691358024691358
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "output_path / \"somefolder/randomfile-2.txt\")",
+      "got": "output_path / \"randomfile-2.txt\")",
+      "got_raw": "output_path / \"randomfile-2.txt\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"icon\" href=\"favicon-3.ico\">')",
+      "got": "\"favicon.ico\"",
+      "got_raw": "\"favicon.ico\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.48148148148148145
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"icon\" href=\"assets/favicon.ico\">')",
+      "got": "\"favicon.png\"",
+      "got_raw": "\"favicon.png\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.3389830508474576
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"icon\" href=\"../assets/favicon.ico\">')",
+      "got": "\"assets/favicon.ico\"",
+      "got_raw": "\"assets/favicon.ico\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.5797101449275363
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"icon\" href=\"../assets-2/favicon-2.ico\">')",
+      "got": "\"favicon-3.ico\"",
+      "got_raw": "\"favicon-3.ico\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.4117647058823529
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"icon\" href=\"https://hogenttin.github.io/cdn/favicon/favicon.ico\">')",
+      "got": "\"favicon\"",
+      "got_raw": "\"favicon\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.20454545454545456
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"icon\" href=\"favicon-2.ico\">')",
+      "got": "\"assets/favicon.ico\"",
+      "got_raw": "\"assets/favicon.ico\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.45901639344262296
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"icon\" href=\"assets/favicon.ico\">')",
+      "got": "\"slides\"",
+      "got_raw": "\"slides\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.18518518518518517
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"stylesheet\" href=\"theme-2.css\" />')",
+      "got": "\"favicon-2.ico\"",
+      "got_raw": "\"favicon-2.ico\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.22580645161290322
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"icon\" href=\"../assets/favicon.ico\">')",
+      "got": "\"assets/someimage.png\"",
+      "got_raw": "\"assets/someimage.png\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.30985915492957744
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<span class=\"node-title\">frontmatter title</span>')",
+      "got": "\"1.0.0\"",
+      "got_raw": "\"1.0.0\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.06779661016949153
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"stylesheet\" href=\"assets/theme.css\" />')",
+      "got": "\"assets/favicon.ico\"",
+      "got_raw": "\"assets/favicon.ico\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.2777777777777778
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "output_path / \"mkslides-assets/highlight-js-themes/vs.css\")",
+      "got": "output_path / \"mkslides-assets/reveal-js/dist/theme/solarized.js\")",
+      "got_raw": "output_path / \"mkslides-assets/reveal-js/dist/theme/solarized.js\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.736
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "output_path / \"mkslides-assets/reveal-js/dist/theme/solarized.css\")",
+      "got": "\"frontmatter title\"",
+      "got_raw": "\"frontmatter title\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.23255813953488372
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"stylesheet\" href=\"mkslides-assets/highlight-js-themes/vs.css\" />')",
+      "got": "\"vs.css\")",
+      "got_raw": "\"vs.css\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.20689655172413793
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"stylesheet\" href=\"mkslides-assets/reveal-js/dist/theme/solarized.css\" />')",
+      "got": "\"1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+      "got_raw": "\"1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.009345794392523364
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "output_path / \"example-4.png\")",
+      "got": "output_path / \"img/somefolder/example-4.png\")",
+      "got_raw": "output_path / \"img/somefolder/example-4.png\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "output_path / \"img/example-1.png\")",
+      "got": "output_path / \"slides\")",
+      "got_raw": "output_path / \"slides\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "output_path / \"somefolder/example-2.png\")",
+      "got": "output_path / \"img/example-2.png\")",
+      "got_raw": "output_path / \"img/example-2.png\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.8533333333333334
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "output_path / \"img/somefolder/example-3.png\")",
+      "got": "output_path / \"img/example-3.png\")",
+      "got_raw": "output_path / \"img/example-3.png\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.8607594936708861
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "\"<h1>Lorem ipsum</h1>\")",
+      "got": "\"Lorem ipsum\"",
+      "got_raw": "\"Lorem ipsum\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.7222222222222222
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "\"<title>Lorem ipsum</title>\")",
+      "got": "\"Index\")",
+      "got_raw": "\"Index\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.21621621621621623
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'Documentation built with <a href=\"https://martenbe.github.io/mkslides/\" target=\"_blank\">MkSlides</a>.')",
+      "got": "\"index.html\"",
+      "got_raw": "\"index.html\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.08620689655172414
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "re.compile( r\"\"\" <section\\s+data-markdown .*? \\s+data-separator=\"\\^\\\\s\\*---\\\\s\\*\\$\"\\s+ .*? \\s+data-separator-vertical=\"\\^\\\\s\\*-v-\\\\s\\*\\$\"\\s+ .*? \\s+data-separator-notes=\"\\^Notes\\?:\"\\s+ .*? \\s+data-charset=\"utf-8\"\\s+ .*? > \"\"\", re.VERBOSE | re.DOTALL, ))",
+      "got": "\"revealjs-markdown-data-options\"",
+      "got_raw": "\"revealjs-markdown-data-options\"",
+      "exact_match": false,
+      "code_bleu": 0.007575757575757576,
+      "edit_similarity": 0.0912280701754386
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "expected_returncode",
+      "got": "expected_returncode",
+      "got_raw": "expected_returncode",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'menu: {\"openButton\": true, \"openOnInit\": true}')",
+      "got": "\"",
+      "got_raw": "\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.04
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<script src=\"https://cdn.jsdelivr.net/npm/reveal.js-menu@2.1.0/menu.min.js\"></script>')",
+      "got": "\"",
+      "got_raw": "\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.02247191011235955
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"stylesheet\" href=\"https://cdn.jsdelivr.net/npm/reveal.js-menu@2.1.0/menu.min.css\" />')",
+      "got": "\"1.0.0\"",
+      "got_raw": "\"1.0.0\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.11428571428571428
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<script src=\"https://cdn.jsdelivr.net/npm/reveal-plantuml/dist/reveal-plantuml.min.js\"></script>')",
+      "got": "\"",
+      "got_raw": "\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.02
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<script src=\"https://cdn.jsdelivr.net/npm/reveal.js-mermaid-plugin/plugin/mermaid/mermaid.min.js\"></script>')",
+      "got": "\"",
+      "got_raw": "\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.018018018018018018
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "re.compile( r\"\"\" aaaaa-aaaa .*? aaa-aa .*? a \"\"\", re.VERBOSE | re.DOTALL, ))",
+      "got": "\"preprocessing\"",
+      "got_raw": "\"preprocessing\"",
+      "exact_match": false,
+      "code_bleu": 0.02631578947368421,
+      "edit_similarity": 0.10989010989010989
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "re.compile( r\"\"\" bbbbb-bbbb .*? bbb-bb .*? b \"\"\", re.VERBOSE | re.DOTALL, ))",
+      "got": "\"aaaa-aaaa\\n\"",
+      "got_raw": "\"aaaa-aaaa\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.02631578947368421,
+      "edit_similarity": 0.06741573033707865
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "expected_returncode",
+      "got": "expected_returncode",
+      "got_raw": "expected_returncode",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "\"[](/folder/test.md)\")",
+      "got": "\"test.md\"",
+      "got_raw": "\"test.md\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.5806451612903226
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "\"[](someslides-2.html)\")",
+      "got": "\"1\")",
+      "got_raw": "\"1\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.21428571428571427
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "\"[](./someslides-2.html)\")",
+      "got": "\"[](someslides-3.html)\")",
+      "got_raw": "\"[](someslides-3.html)\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.92
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "\"[test](/folder/test.md)\")",
+      "got": "\"/folder/test.md\"",
+      "got_raw": "\"/folder/test.md\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.7906976744186046
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "\"[test](someslides-2.html)\")",
+      "got": "\"[](../someslides-2.html)\")",
+      "got_raw": "\"[](../someslides-2.html)\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.8727272727272727
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "\"history: true,\")",
+      "got": "\"Hello world!\")",
+      "got_raw": "\"Hello world!\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.3125
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "re.compile( r\"\"\" Reveal\\.initialize\\({ .*? transition:\\s+\"fade\", .*? }\\); \"\"\", re.VERBOSE | re.DOTALL, ))",
+      "got": "\"revealjs_string_options\"",
+      "got_raw": "\"revealjs_string_options\"",
+      "exact_match": false,
+      "code_bleu": 0.023809523809523808,
+      "edit_similarity": 0.23076923076923078
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "re.compile( r\"\"\" Reveal\\.initialize\\({ .*? height:\\s+1080, .*? width:\\s+1920, .*? }\\); \"\"\", re.VERBOSE | re.DOTALL, ))",
+      "got": "\"revealjs_integer_options.html\"",
+      "got_raw": "\"revealjs_integer_options.html\"",
+      "exact_match": false,
+      "code_bleu": 0.02631578947368421,
+      "edit_similarity": 0.18791946308724833
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "\"<h1>Index</h1>\")",
+      "got": "\"Slides",
+      "got_raw": "\"Slides 2\")",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "output_path / \"index.html\")",
+      "got": "output_path / \"slides.html\")",
+      "got_raw": "output_path / \"slides.html\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "\"Lorem ipsum dolor sit amet\")",
+      "got": "\"Hello World\")",
+      "got_raw": "\"Hello World\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.32558139534883723
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "output_path / \"someslides-1.html\")",
+      "got": "output_path / \"index.md\")",
+      "got_raw": "output_path / \"index.md\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.7457627118644068
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "output_path / \"someslides-2.html\")",
+      "got": "output_path / \"someslides-2.html\")",
+      "got_raw": "output_path / \"someslides-2.html\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "\"\ud83d\udc4d\")",
+      "got": "\"\ud83d\ude00\")",
+      "got_raw": "\"\ud83d\ude00\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "\"\u26a0\ufe0f\")",
+      "got": "\"Hello",
+      "got_raw": "\"Hello World\")",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.18181818181818182
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"stylesheet\" href=\"theme-3.css\" />')",
+      "got": "\"assets/theme.css\")",
+      "got_raw": "\"assets/theme.css\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.48484848484848486
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"stylesheet\" href=\"assets/theme.css\" />')",
+      "got": "\"mkslides-assets/reveal-js/dist/theme/black.css\"",
+      "got_raw": "\"mkslides-assets/reveal-js/dist/theme/black.css\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.42
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"stylesheet\" href=\"../assets/theme.css\" />')",
+      "got": "\"../assets/theme.css\"",
+      "got_raw": "\"../assets/theme.css\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.5526315789473685
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"stylesheet\" href=\"highlight-theme-3.css\" />')",
+      "got": "True(assert_file_exist(output_path / \"someslides-1.html\"))",
+      "got_raw": "True(assert_file_exist(output_path / \"someslides-1.html\"))",
+      "exact_match": false,
+      "code_bleu": 0.14113991930789777,
+      "edit_similarity": 0.12173913043478261
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"stylesheet\" href=\"../assets-2/theme-2.css\" />')",
+      "got": "\"theme-4.css\"",
+      "got_raw": "\"theme-4.css\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "output_path / \"mkslides-assets/reveal-js/dist/theme/simple.css\")",
+      "got": "output_path / \"index.html\")",
+      "got_raw": "output_path / \"index.html\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.46153846153846156
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"stylesheet\" href=\"https://example.org/theme.css\" />')",
+      "got": "\"mkslides-assets/reveal-js/dist/reveal.css\"",
+      "got_raw": "\"mkslides-assets/reveal-js/dist/reveal.css\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.37037037037037035
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"stylesheet\" href=\"https://example.org/highlight-theme.css\" />')",
+      "got": "\"example.org/theme.css\"",
+      "got_raw": "\"example.org/theme.css\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.46938775510204084
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"stylesheet\" href=\"mkslides-assets/reveal-js/dist/reveal.css\" />')",
+      "got": "\"absolute_url_slideshow_theme_path.html\"",
+      "got_raw": "\"absolute_url_slideshow_theme_path.html\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"stylesheet\" href=\"mkslides-assets/highlight-js-themes/vs.css\" />')",
+      "got": "\"mkslides-assets/reveal-js/dist/theme/simple.css\")",
+      "got_raw": "\"mkslides-assets/reveal-js/dist/theme/simple.css\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.53125
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"stylesheet\" href=\"../mkslides-assets/reveal-js/dist/reveal.css\" />')",
+      "got": "\"mkslides-assets/highlight-js-themes/vs.css\")",
+      "got_raw": "\"mkslides-assets/highlight-js-themes/vs.css\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.48
+    },
+    {
+      "repo": "MartenBE/mkslides",
+      "expected": "'<link rel=\"stylesheet\" href=\"../mkslides-assets/highlight-js-themes/vs.css\" />')",
+      "got": "\"mkslides-assets/highlight-js-themes/vs.css\")",
+      "got_raw": "\"mkslides-assets/highlight-js-themes/vs.css\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.7142857142857143
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "0)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "30)",
+      "got": "20)",
+      "got_raw": "20)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "any(\"SUCCESS\" in msg for msg in messages))",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.08214563669180407,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "any(\"FAILURE\" in msg for msg in messages))",
+      "got": "any(\"ERROR\" in msg for msg in messages))",
+      "got_raw": "any(\"ERROR\" in msg for msg in messages))",
+      "exact_match": false,
+      "code_bleu": 0.7691605673134586,
+      "edit_similarity": 0.8780487804878049
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "any(\"Aborting\" in msg for msg in messages))",
+      "got": "len(messages)",
+      "got_raw": "len(messages)",
+      "exact_match": false,
+      "code_bleu": 0.18477554478349628,
+      "edit_similarity": 0.39285714285714285
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "any(\"FAILURE\" in msg and \"height\" in msg for msg in messages))",
+      "got": "messages)",
+      "got_raw": "messages)",
+      "exact_match": false,
+      "code_bleu": 0.09428090415820634,
+      "edit_similarity": 0.2535211267605634
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "any(\"FAILURE\" in msg and \"intersection\" in msg for msg in messages))",
+      "got": "messages)",
+      "got_raw": "messages)",
+      "exact_match": false,
+      "code_bleu": 0.09428090415820634,
+      "edit_similarity": 0.23376623376623376
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"refusal\"",
+      "got": "\"Invalid",
+      "got_raw": "\"Invalid session\"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.35294117647058826
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"end_turn\"",
+      "got": "\"stop\"",
+      "got_raw": "\"stop\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.375
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "-1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"running\"",
+      "got": "\"running\"",
+      "got_raw": "\"running\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "result.stdout",
+      "got": "result.output",
+      "got_raw": "result.output",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.7692307692307693
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "result.stderr",
+      "got": "result.stderr",
+      "got_raw": "result.stderr",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "result.error.lower()",
+      "got": "result.error",
+      "got_raw": "result.error",
+      "exact_match": false,
+      "code_bleu": 0.36038417796587763,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "LLMProvider.OPENAI",
+      "got": "LLMProvider.OPENAI",
+      "got_raw": "LLMProvider.OPENAI",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "LLMProvider.ANTHROPIC",
+      "got": "LLMProvider.ANTHROPIC",
+      "got_raw": "LLMProvider.ANTHROPIC",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"get_weather\"",
+      "got": "\"get_weather\"",
+      "got_raw": "\"get_weather\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "3",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "skill.content",
+      "got": "skill.content",
+      "got_raw": "skill.content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "5.0",
+      "got": "5.0",
+      "got_raw": "5.0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "15.0",
+      "got": "15.0",
+      "got_raw": "15.0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "90.0",
+      "got": "90.0",
+      "got_raw": "90.0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "10.0",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "60.0",
+      "got": "30.0",
+      "got_raw": "30.0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "30.0",
+      "got": "30.0",
+      "got_raw": "30.0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "20.0",
+      "got": "20.0",
+      "got_raw": "20.0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "result.content",
+      "got": "result.content",
+      "got_raw": "result.content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "6",
+      "got": "6",
+      "got_raw": "6",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "5",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"user\"",
+      "got": "\"user\"",
+      "got_raw": "\"user\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"system\"",
+      "got": "\"system\"",
+      "got_raw": "\"system\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "agent.messages",
+      "got": "agent.messages",
+      "got_raw": "agent.messages",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "result2.content",
+      "got": "result2.output",
+      "got_raw": "result2.output",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.7586206896551724
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "len(agent.messages)",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"MIT\"",
+      "got": "\"MIT\"",
+      "got_raw": "\"MIT\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"1.0\"",
+      "got": "\"1.0\"",
+      "got_raw": "\"1.0\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "prompt",
+      "got": "prompt",
+      "got_raw": "prompt",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"test-skill\"",
+      "got": "\"test-skill\"",
+      "got_raw": "\"test-skill\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "skill.content",
+      "got": "skill.content",
+      "got_raw": "skill.content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"Test Author\"",
+      "got": "\"Test Author\"",
+      "got_raw": "\"Test Author\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"get_skill\"",
+      "got": "\"get_skill\"",
+      "got_raw": "\"get_skill\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "result.content",
+      "got": "result.output",
+      "got_raw": "result.output",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.7407407407407407
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "8",
+      "got": "7",
+      "got_raw": "7",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "2",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "4",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "6",
+      "got": "characters",
+      "got_raw": " characters",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "7",
+      "got": "",
+      "got_raw": " # 4 + 1",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "3",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "9",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "schema",
+      "got": "schema",
+      "got_raw": "schema",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"object\"",
+      "got": "\"dict\"",
+      "got_raw": "\"dict\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "[\"query\"]",
+      "got": "[\"query\"]",
+      "got_raw": "[\"query\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"function\"",
+      "got": "\"string\"",
+      "got_raw": "\"string\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5555555555555556
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "status_prop",
+      "got": "status_prop",
+      "got_raw": "status_prop",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "[\"location\"]",
+      "got": "[\"location\"]",
+      "got_raw": "[\"location\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"calculator\"",
+      "got": "\"calculate\"",
+      "got_raw": "\"calculate\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8695652173913043
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"get_weather\"",
+      "got": "\"get_weather\"",
+      "got_raw": "\"get_weather\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"Weather data\"",
+      "got": "\"2023-01-01 12:00:00\"",
+      "got_raw": "\"2023-01-01 12:00:00\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.17142857142857143
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"search_database\"",
+      "got": "\"Search\"",
+      "got_raw": "\"Search\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.56
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "result.content",
+      "got": "result.output",
+      "got_raw": "result.output",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.7407407407407407
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"Test content\"",
+      "got": "\"Test content\"",
+      "got_raw": "\"Test content\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MiniMax-AI/Mini-Agent",
+      "expected": "\"Hello, Agent!\"",
+      "got": "\"Hello, Agent!\"",
+      "got_raw": "\"Hello, Agent!\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "0.1)",
+      "got": "0.0001)",
+      "got_raw": "0.0001)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.7272727272727273
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "\"num2\")",
+      "got": "df",
+      "got_raw": "df, \"num2\")",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "\"expected_name\")",
+      "got": "\"expected_name\")",
+      "got_raw": "\"expected_name\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "ColumnsNotEqualError)",
+      "got": "ColumnsNotEqualError)",
+      "got_raw": "ColumnsNotEqualError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "df2)",
+      "got": "df2)",
+      "got_raw": "df2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "0.1)",
+      "got": "ignore_columns=[\"expected_name\"])",
+      "got_raw": "ignore_columns=[\"expected_name\"])",
+      "exact_match": false,
+      "code_bleu": 0.22089591134157885,
+      "edit_similarity": 0.05405405405405406
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "ValueError)",
+      "got": "ValueError",
+      "got_raw": "ValueError, match=\".*invalid.*\")",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "df2.collect())",
+      "got": "df2.collect())",
+      "got_raw": "df2.collect())",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "str(w[-1].message)",
+      "got": "w[-1].message.args[0]",
+      "got_raw": "w[-1].message.args[0]",
+      "exact_match": false,
+      "code_bleu": 0.5593684915933074,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "\"\\033[34mtest\\033[31m\"",
+      "got": "\"test\"",
+      "got_raw": "\"test\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.42857142857142855
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "DataFramesNotEqualError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.45714285714285713
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "\"\\033[97m\\033[4mtest\\033[0m\\033[31m\"",
+      "got": "\"test\"",
+      "got_raw": "\"test\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "df2)",
+      "got": "df2)",
+      "got_raw": "df2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "0.1)",
+      "got": "0.01)",
+      "got_raw": "0.01)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "expected_df)",
+      "got": "expected_df)",
+      "got_raw": "expected_df)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "df2.collect())",
+      "got": "df2.collect())",
+      "got_raw": "df2.collect())",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "\"expected_name\")",
+      "got": "\"expected_name\")",
+      "got_raw": "\"expected_name\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "ColumnsNotEqualError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "SchemasNotEqualError)",
+      "got": "SchemasNotEqualError)",
+      "got_raw": "SchemasNotEqualError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "df2.collect())",
+      "got": "df2.collect())",
+      "got_raw": "df2.collect())",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "DataFramesNotEqualError)",
+      "got": "DataFramesNotEqualError)",
+      "got_raw": "DataFramesNotEqualError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "s2)",
+      "got": "s2)",
+      "got_raw": "s2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "Color.BLUE",
+      "got": "Color.BLUE",
+      "got_raw": "Color.BLUE",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "[Style.BOLD, Style.UNDERLINE]",
+      "got": "[\"bold\", \"underline\"]",
+      "got_raw": "[\"bold\", \"underline\"]",
+      "exact_match": false,
+      "code_bleu": 0.18037106415283274,
+      "edit_similarity": 0.24
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "\"All elements in the list must be strings\"",
+      "got": "\"Format values must be strings\"",
+      "got_raw": "\"Format values must be strings\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6301369863013698
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "\"Invalid style name: invalid_style. Valid style names are ['bold', 'underline', 'blink', 'invert', 'hide']\"",
+      "got": "\"Invalid style: invalid_style\"",
+      "got_raw": "\"Invalid style: invalid_style\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.43795620437956206
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "\"Invalid color name: invalid_color. Valid color names are \" \"['black', 'red', 'green', 'yellow', 'blue', 'purple', 'cyan', 'light_gray', \" \"'dark_gray', 'light_red', 'light_green', 'light_yellow', 'light_blue', 'light_purple', \" \"'light_cyan', 'white']\"",
+      "got": "\"Invalid color: invalid_color\"",
+      "got_raw": "\"Invalid color: invalid_color\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.2049469964664311
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "\"Invalid value: invalid_color. Valid values are colors: \" \"['black', 'red', 'green', 'yellow', 'blue', 'purple', 'cyan', 'light_gray', \" \"'dark_gray', 'light_red', 'light_green', 'light_yellow', 'light_blue', 'light_purple', \" \"'light_cyan', 'white'] and styles: ['bold', 'underline', 'blink', 'invert', 'hide']\"",
+      "got": "\"Invalid color: invalid_color\"",
+      "got_raw": "\"Invalid color: invalid_color\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.14035087719298245
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "\"Invalid value: invalid_style. Valid values are colors: \" \"['black', 'red', 'green', 'yellow', 'blue', 'purple', 'cyan', 'light_gray', \" \"'dark_gray', 'light_red', 'light_green', 'light_yellow', 'light_blue', 'light_purple', \" \"'light_cyan', 'white'] and styles: ['bold', 'underline', 'blink', 'invert', 'hide']\"",
+      "got": "\"Invalid style 'invalid_style'\"",
+      "got_raw": "\"Invalid style 'invalid_style'\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.052478134110787174
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "None",
+      "got": "Style.BOLD",
+      "got_raw": "Style.BOLD",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.14285714285714285
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "Color.RED",
+      "got": "Color.RED",
+      "got_raw": "Color.RED",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "Color.BLUE",
+      "got": "Color.DEFAULT",
+      "got_raw": "Color.DEFAULT",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.6086956521739131
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "Color.CYAN",
+      "got": "\"cyan\"",
+      "got_raw": "\"cyan\"",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "Color.GREEN",
+      "got": "\"green\"",
+      "got_raw": "\"green\"",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.1111111111111111
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "Color.YELLOW",
+      "got": "\"yellow\"",
+      "got_raw": "\"yellow\"",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "Color.PURPLE",
+      "got": "\"purple\"",
+      "got_raw": "\"purple\"",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.1
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "[Style.BLINK]",
+      "got": "[\"blink\"]",
+      "got_raw": "[\"blink\"]",
+      "exact_match": false,
+      "code_bleu": 0.29999999999999993,
+      "edit_similarity": 0.2727272727272727
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "[Style.UNDERLINE]",
+      "got": "Style.BOLD",
+      "got_raw": "Style.BOLD",
+      "exact_match": false,
+      "code_bleu": 0.3567621345008163,
+      "edit_similarity": 0.5185185185185185
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "[Style.INVERT, Style.HIDE]",
+      "got": "[\"invert\", \"hide\"]",
+      "got_raw": "[\"invert\", \"hide\"]",
+      "exact_match": false,
+      "code_bleu": 0.18037106415283274,
+      "edit_similarity": 0.2727272727272727
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "[Style.BOLD, Style.UNDERLINE]",
+      "got": "[\"bold\", \"underline\"]",
+      "got_raw": "[\"bold\", \"underline\"]",
+      "exact_match": false,
+      "code_bleu": 0.18037106415283274,
+      "edit_similarity": 0.24
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "\"Invalid style name: invalid_style. Valid style names are ['bold', 'underline', 'blink', 'invert', 'hide']\"",
+      "got": "\"Invalid style: invalid_style\"",
+      "got_raw": "\"Invalid style: invalid_style\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.43795620437956206
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "\"Invalid color name: invalid_color. Valid color names are \" \"['black', 'red', 'green', 'yellow', 'blue', 'purple', 'cyan', 'light_gray', \" \"'dark_gray', 'light_red', 'light_green', 'light_yellow', 'light_blue', 'light_purple', \" \"'light_cyan', 'white']\"",
+      "got": "\"Invalid color: invalid_color\"",
+      "got_raw": "\"Invalid color: invalid_color\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.2049469964664311
+    },
+    {
+      "repo": "MrPowers/chispa",
+      "expected": "expected_string",
+      "got": "expected_string",
+      "got_raw": "expected_string",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "beta)",
+      "got": "beta)",
+      "got_raw": "beta)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "alpha)",
+      "got": "alpha)",
+      "got_raw": "alpha)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "greedy)",
+      "got": "greedy)",
+      "got_raw": "greedy)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "better_history)",
+      "got": "better_history)",
+      "got_raw": "better_history)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "lookback_period)",
+      "got": "lookback_period)",
+      "got_raw": "lookback_period)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "(ValueError, TypeError))",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.6285714285714286
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "eta)",
+      "got": "eta)",
+      "got_raw": "eta)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "gamma)",
+      "got": "gamma)",
+      "got_raw": "gamma)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "[7100])",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": false,
+      "code_bleu": 0.47920732818470435,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "[7100, 7200])",
+      "got": "[7100, 7200])",
+      "got_raw": "[7100, 7200])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "[7100, 7200, 7120])",
+      "got": "[7100, 7200, 7120])",
+      "got_raw": "[7100, 7200, 7120])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "[7100, 7200, 7120, 7100])",
+      "got": "[7100, 7200, 7120, 7100])",
+      "got_raw": "[7100, 7200, 7120, 7100])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "[7200, 7120, 7100, 7200])",
+      "got": "[7100, 7200, 7120, 7100, 7200])",
+      "got_raw": "[7100, 7200, 7120, 7100, 7200])",
+      "exact_match": false,
+      "code_bleu": 0.7539221180326288,
+      "edit_similarity": 0.8928571428571429
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "beta)",
+      "got": "beta)",
+      "got_raw": "beta)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "alpha)",
+      "got": "alpha)",
+      "got_raw": "alpha)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "gamma)",
+      "got": "gamma)",
+      "got_raw": "gamma)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "delta)",
+      "got": "delta)",
+      "got_raw": "delta)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "end)",
+      "got": "end)",
+      "got_raw": "end)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "step)",
+      "got": "step)",
+      "got_raw": "step)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "start)",
+      "got": "start)",
+      "got_raw": "start)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "method)",
+      "got": "method)",
+      "got_raw": "method)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "end)",
+      "got": "end)",
+      "got_raw": "end)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "step)",
+      "got": "step)",
+      "got_raw": "step)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "start)",
+      "got": "start)",
+      "got_raw": "start)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "method)",
+      "got": "method)",
+      "got_raw": "method)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "exp_end)",
+      "got": "exp_end)",
+      "got_raw": "exp_end)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "exp_step)",
+      "got": "exp_step)",
+      "got_raw": "exp_step)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "exp_start)",
+      "got": "exp_start)",
+      "got_raw": "exp_start)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "end)",
+      "got": "end)",
+      "got_raw": "end)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "step)",
+      "got": "step)",
+      "got_raw": "step)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "start)",
+      "got": "start)",
+      "got_raw": "start)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "sa_end)",
+      "got": "sa_end)",
+      "got_raw": "sa_end)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "sa_step)",
+      "got": "sa_step)",
+      "got_raw": "sa_step)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "sa_start)",
+      "got": "sa_start)",
+      "got_raw": "sa_start)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "\"linear\")",
+      "got": "\"linear\")",
+      "got_raw": "\"linear\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "\"exponential\")",
+      "got": "\"autofit\")",
+      "got_raw": "\"autofit\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "update(1, 0.5, uc))",
+      "got": "update(1, 0.5, uc))",
+      "got_raw": "update(1, 0.5, uc))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "alpha)",
+      "got": "alpha)",
+      "got_raw": "alpha)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "scores)",
+      "got": "scores)",
+      "got_raw": "scores)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "(0, 0))",
+      "got": "(0, 0))",
+      "got_raw": "(0, 0))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "(1, 0))",
+      "got": "(1, 0))",
+      "got_raw": "(1, 0))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "(0, 0))",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": false,
+      "code_bleu": 0.23570226039551584,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "(1, 0))",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": false,
+      "code_bleu": 0.23570226039551584,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "[5, 0, 3, 0])",
+      "got": "[0, 0, 0, 0])",
+      "got_raw": "[0, 0, 0, 0])",
+      "exact_match": false,
+      "code_bleu": 0.4832697830906221,
+      "edit_similarity": 0.8461538461538461
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "AttributeError)",
+      "got": "AttributeError)",
+      "got_raw": "AttributeError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "0)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "0.25)",
+      "got": "[0.25",
+      "got_raw": "[0.25, 0.25, 0.25, 0.25])",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "1 / 3)",
+      "got": "1 / 3)",
+      "got_raw": "1 / 3)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "decay)",
+      "got": "decay)",
+      "got_raw": "decay)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "scores)",
+      "got": "scores)",
+      "got_raw": "scores)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "(1, 2))",
+      "got": "(4, 2))",
+      "got_raw": "(4, 2))",
+      "exact_match": false,
+      "code_bleu": 0.6434588841607617,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "num_repair)",
+      "got": "num_repair)",
+      "got_raw": "num_repair)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "num_destroy)",
+      "got": "num_destroy)",
+      "got_raw": "num_destroy)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "op_coupling)",
+      "got": "op_coupling)",
+      "got_raw": "op_coupling)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "expected[0])",
+      "got": "expected[0])",
+      "got_raw": "expected[0])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "expected[1])",
+      "got": "expected[1])",
+      "got_raw": "expected[1])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "seg_length)",
+      "got": "seg_length)",
+      "got_raw": "seg_length)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "expected[0])",
+      "got": "expected[0])",
+      "got_raw": "expected[0])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "expected[1])",
+      "got": "expected[1])",
+      "got_raw": "expected[1])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "max_iterations)",
+      "got": "max_iterations)",
+      "got_raw": "max_iterations)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "max_runtime)",
+      "got": "max_runtime)",
+      "got_raw": "max_runtime)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "max_iterations)",
+      "got": "max_iterations)",
+      "got_raw": "max_iterations)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "0)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "count)",
+      "got": "count)",
+      "got_raw": "count)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "desired)",
+      "got": "desired)",
+      "got_raw": "desired)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "max_runtime)",
+      "got": "max_runtime)",
+      "got_raw": "max_runtime)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "max_iterations)",
+      "got": "max_iterations",
+      "got_raw": "max_iterations + 1)",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.9655172413793104
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "\"repair_operator\")",
+      "got": "\"repair_operator\")",
+      "got_raw": "\"repair_operator\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "\"destroy_operator\")",
+      "got": "destroy_operator.__name__)",
+      "got_raw": "destroy_operator.__name__)",
+      "exact_match": false,
+      "code_bleu": 0.35930411196308426,
+      "edit_similarity": 0.7555555555555555
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "max_iterations + 1)",
+      "got": "max_iterations + 1)",
+      "got_raw": "max_iterations + 1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "0)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "1)",
+      "got": "[time",
+      "got_raw": "[time for time in range(1, 100)])",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "99)",
+      "got": "100)",
+      "got_raw": "100)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "count)",
+      "got": "count)",
+      "got_raw": "count)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "N-Wouda/ALNS",
+      "expected": "objective)",
+      "got": "objective)",
+      "got_raw": "objective)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "0)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "3)",
+      "got": "3)",
+      "got_raw": "3)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "4)",
+      "got": "4)",
+      "got_raw": "4)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "2)",
+      "got": "2)",
+      "got_raw": "2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "0)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "1)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "3)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "2)",
+      "got": "int(ResourceType.file))",
+      "got_raw": "int(ResourceType.file))",
+      "exact_match": false,
+      "code_bleu": 0.18575057999133598,
+      "edit_similarity": 0.08
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "3)",
+      "got": "3)",
+      "got_raw": "3)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "4)",
+      "got": "4)",
+      "got_raw": "4)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "0)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "0)",
+      "got": "0o000)",
+      "got_raw": "0o000)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "0)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "3)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "0)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "5)",
+      "got": "5)",
+      "got_raw": "5)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "7)",
+      "got": "7)",
+      "got_raw": "7)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"\")",
+      "got": "\"\")",
+      "got_raw": "\"\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"\")",
+      "got": "\"\")",
+      "got_raw": "\"\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"\")",
+      "got": "\"\")",
+      "got_raw": "\"\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"w\")",
+      "got": "\"Mode('w')\")",
+      "got_raw": "\"Mode('w')\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "b\"f\")",
+      "got": "b\"o\")",
+      "got_raw": "b\"o\")",
+      "exact_match": false,
+      "code_bleu": 0.49999999999999994,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"gz\")",
+      "got": "\"gz\")",
+      "got_raw": "\"gz\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"bar\")",
+      "got": "\"bar\")",
+      "got_raw": "\"bar\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"baz\")",
+      "got": "\"baz\")",
+      "got_raw": "\"baz\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"foo\")",
+      "got": "\"foo\")",
+      "got_raw": "\"foo\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"baz\")",
+      "got": "\"baz\")",
+      "got_raw": "\"baz\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"foo\")",
+      "got": "\"foo\")",
+      "got_raw": "\"foo\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\".py\")",
+      "got": "\".py\")",
+      "got_raw": "\".py\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\".gz\")",
+      "got": "\".tar.gz\")",
+      "got_raw": "\".tar.gz\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "b\"m3\")",
+      "got": "b\"m3\")",
+      "got_raw": "b\"m3\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "b\"m2\")",
+      "got": "b\"m2\")",
+      "got_raw": "b\"m2\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "b\"m1\")",
+      "got": "b\"m1\")",
+      "got_raw": "b\"m1\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"1.0\")",
+      "got": "\"1.0\")",
+      "got_raw": "\"1.0\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"baz\")",
+      "got": "\"baz\")",
+      "got_raw": "\"baz\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"bar\")",
+      "got": "\"bar\")",
+      "got_raw": "\"bar\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"baz\")",
+      "got": "\"foo/bar/baz\")",
+      "got_raw": "\"foo/bar/baz\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"foo\")",
+      "got": "\"foo\")",
+      "got_raw": "\"foo\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"bar\")",
+      "got": "\"bar\")",
+      "got_raw": "\"bar\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"baz\")",
+      "got": "\"baz\")",
+      "got_raw": "\"baz\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "b\"baz\")",
+      "got": "b\"baz\")",
+      "got_raw": "b\"baz\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "b\"foo\")",
+      "got": "\"foo\")",
+      "got_raw": "\"foo\")",
+      "exact_match": false,
+      "code_bleu": 0.4714045207910317,
+      "edit_similarity": 0.9230769230769231
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"will\")",
+      "got": "\"will\")",
+      "got_raw": "\"will\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"devs\")",
+      "got": "\"devs\")",
+      "got_raw": "\"devs\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "b\"foo\")",
+      "got": "b\"foo\")",
+      "got_raw": "b\"foo\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"foo/\")",
+      "got": "\"foo\")",
+      "got_raw": "\"foo\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.9230769230769231
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"/bar\")",
+      "got": "\"bar\")",
+      "got_raw": "\"bar\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.9230769230769231
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"hello\")",
+      "got": "b\"hello\")",
+      "got_raw": "b\"hello\")",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.9411764705882353
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"foofoo\")",
+      "got": "\"foofoo\")",
+      "got_raw": "\"foofoo\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"1 byte\")",
+      "got": "\"1 byte\")",
+      "got_raw": "\"1 byte\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"1.0 KB\")",
+      "got": "\"1.0 kB\")",
+      "got_raw": "\"1.0 kB\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"1.0 MB\")",
+      "got": "\"1.0 MB\")",
+      "got_raw": "\"1.0 MB\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"1.5 MB\")",
+      "got": "\"1.5 MB\")",
+      "got_raw": "\"1.5 MB\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"1.0 kB\")",
+      "got": "\"1000 bytes\")",
+      "got_raw": "\"1000 bytes\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5454545454545454
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"1.2 MB\")",
+      "got": "\"1.2 MB\")",
+      "got_raw": "\"1.2 MB\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"bar.py\")",
+      "got": "\"bar.py\")",
+      "got_raw": "\"bar.py\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "b\"Hello\")",
+      "got": "b\"Hello\")",
+      "got_raw": "b\"Hello\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"foobar\")",
+      "got": "\"foobar\")",
+      "got_raw": "\"foobar\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"Source\")",
+      "got": "\"Source\")",
+      "got_raw": "\"Source\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "b\"foofoo\")",
+      "got": "b\"foofoo\")",
+      "got_raw": "b\"foofoo\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"oh dear\")",
+      "got": "\"oh dear\")",
+      "got_raw": "\"oh dear\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"0 bytes\")",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "\"2 bytes\")",
+      "got": "\"2 bytes\")",
+      "got_raw": "\"2 bytes\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "p)",
+      "got": "p.names)",
+      "got_raw": "p.names)",
+      "exact_match": false,
+      "code_bleu": 0.3976353643835253,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "m2)",
+      "got": "m2)",
+      "got_raw": "m2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "pp)",
+      "got": "pp)",
+      "got_raw": "pp)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "fs)",
+      "got": "os.path.exists(fs._tar_file))",
+      "got_raw": "os.path.exists(fs._tar_file))",
+      "exact_match": false,
+      "code_bleu": 0.12605968092174913,
+      "edit_similarity": 0.1875
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "bool)",
+      "got": "bool)",
+      "got_raw": "bool)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "list)",
+      "got": "list)",
+      "got_raw": "list)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "OSFS)",
+      "got": "OSFS)",
+      "got_raw": "OSFS)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "data1)",
+      "got": "data1)",
+      "got_raw": "data1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "data2)",
+      "got": "data2)",
+      "got_raw": "data2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "data3)",
+      "got": "data3)",
+      "got_raw": "data3)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "data4)",
+      "got": "data4)",
+      "got_raw": "data4)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "FTPFS)",
+      "got": "FTPFS)",
+      "got_raw": "FTPFS)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "SubFS)",
+      "got": "FTPFS)",
+      "got_raw": "FTPFS)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "bytes)",
+      "got": "bytes)",
+      "got_raw": "bytes)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "NoURL)",
+      "got": "NoURL)",
+      "got_raw": "NoURL)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "NoURL)",
+      "got": "NoURL)",
+      "got_raw": "NoURL)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "copied)",
+      "got": "copied)",
+      "got_raw": "copied)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "levels)",
+      "got": "levels)",
+      "got_raw": "levels)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "parsed)",
+      "got": "parsed)",
+      "got_raw": "parsed)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "result)",
+      "got": "result)",
+      "got_raw": "result)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "IOError)",
+      "got": "IOError)",
+      "got_raw": "IOError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "cleanup)",
+      "got": "cleanup)",
+      "got_raw": "cleanup)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "FSError)",
+      "got": "FSError)",
+      "got_raw": "FSError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "OSError)",
+      "got": "errors.FSError)",
+      "got_raw": "errors.FSError)",
+      "exact_match": false,
+      "code_bleu": 0.35930411196308426,
+      "edit_similarity": 0.6086956521739131
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "FSError)",
+      "got": "FSError)",
+      "got_raw": "FSError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "MemoryFS)",
+      "got": "MemoryFS)",
+      "got_raw": "MemoryFS)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "MemoryFS)",
+      "got": "MemoryFS)",
+      "got_raw": "MemoryFS)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "contents)",
+      "got": "contents)",
+      "got_raw": "contents)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "text_type)",
+      "got": "text_type)",
+      "got_raw": "text_type)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "TypeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "exception)",
+      "got": "exception)",
+      "got_raw": "exception)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "TypeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "0o777)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "0o755)",
+      "got": "0o755)",
+      "got_raw": "0o755)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "0o700)",
+      "got": "0o644)",
+      "got_raw": "0o644)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "0o764)",
+      "got": "493)",
+      "got_raw": "493)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "p.u_r)",
+      "got": "p.read)",
+      "got_raw": "p.read)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.6153846153846154
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "args[0])",
+      "got": "exc.__class__)",
+      "got_raw": "exc.__class__)",
+      "exact_match": false,
+      "code_bleu": 0.28744328957046744,
+      "edit_similarity": 0.2727272727272727
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "dates[0])",
+      "got": "dates[0])",
+      "got_raw": "dates[0])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "dates[1])",
+      "got": "dates[1])",
+      "got_raw": "dates[1])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "dates[2])",
+      "got": "dates[2])",
+      "got_raw": "dates[2])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "dates[3])",
+      "got": "dates[3])",
+      "got_raw": "dates[3])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "info.name)",
+      "got": "info.get(\"basic\"",
+      "got_raw": "info.get(\"basic\", \"foo\"))",
+      "exact_match": false,
+      "code_bleu": 0.35930411196308426,
+      "edit_similarity": 0.46153846153846156
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "mode.text)",
+      "got": "mode.text)",
+      "got_raw": "mode.text)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "bar.is_dir)",
+      "got": "bar.is_file)",
+      "got_raw": "bar.is_file)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.782608695652174
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "top.is_dir)",
+      "got": "top.is_dir)",
+      "got_raw": "top.is_dir)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "os.SEEK_CUR)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": false,
+      "code_bleu": 0.2686424829558855,
+      "edit_similarity": 0.14285714285714285
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "os.SEEK_END)",
+      "got": "os.SEEK_END)",
+      "got_raw": "os.SEEK_END)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "os.SEEK_SET)",
+      "got": "os.SEEK_SET)",
+      "got_raw": "os.SEEK_SET)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "ftps_fs.tls)",
+      "got": "ftps_fs.ssl)",
+      "got_raw": "ftps_fs.ssl)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.8333333333333334
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "self.mem_fs)",
+      "got": "self.mem_fs)",
+      "got_raw": "self.mem_fs)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "foo_info.raw)",
+      "got": "foo_info.namespaces)",
+      "got_raw": "foo_info.namespaces)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "bar_info.raw)",
+      "got": "bar_info.raw)",
+      "got_raw": "bar_info.raw)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "errors.NoURL)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.2686424829558855,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "six.text_type)",
+      "got": "six.text_type)",
+      "got_raw": "six.text_type)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "six.text_type)",
+      "got": "str)",
+      "got_raw": "str)",
+      "exact_match": false,
+      "code_bleu": 0.2686424829558855,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "self.lrucache)",
+      "got": "self.lrucache)",
+      "got_raw": "self.lrucache)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "six.text_type)",
+      "got": "six.text_type)",
+      "got_raw": "six.text_type)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "self.server.host)",
+      "got": "self.server.host)",
+      "got_raw": "self.server.host)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "self.fs.features)",
+      "got": "self.server.handler.supported_commands)",
+      "got_raw": "self.server.handler.supported_commands)",
+      "exact_match": false,
+      "code_bleu": 0.239802967618271,
+      "edit_similarity": 0.39285714285714285
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "io.BufferedWriter)",
+      "got": "io.BufferedWriter)",
+      "got_raw": "io.BufferedWriter)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "io.BufferedReader)",
+      "got": "io.BufferedReader)",
+      "got_raw": "io.BufferedReader)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "tarfile.ReadError)",
+      "got": "tarfile.ReadError)",
+      "got_raw": "tarfile.ReadError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "errors.InvalidPath)",
+      "got": "errors.InvalidPath)",
+      "got_raw": "errors.InvalidPath)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "[])",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "[])",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "[])",
+      "got": "[\"foo.py\"",
+      "got_raw": "[\"foo.py\", \"bar.py\", \"baz.py\", \"egg\", \"a\", \"b\", \"r\"])",
+      "exact_match": false,
+      "code_bleu": 0.35818997727451396,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "[])",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "[])",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "[])",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "[])",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "[])",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "[\"/\"])",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": false,
+      "code_bleu": 0.47920732818470435,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "[\"./\"])",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": false,
+      "code_bleu": 0.47920732818470435,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "[\"foo\"])",
+      "got": "[\"foo\"])",
+      "got_raw": "[\"foo\"])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "[\"foo\"])",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": false,
+      "code_bleu": 0.47920732818470435,
+      "edit_similarity": 0.5454545454545454
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "[\"baz\"])",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": false,
+      "code_bleu": 0.47920732818470435,
+      "edit_similarity": 0.5454545454545454
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "(None, None))",
+      "got": "None)",
+      "got_raw": "None)",
+      "exact_match": false,
+      "code_bleu": 0.23570226039551584,
+      "edit_similarity": 0.5555555555555556
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "[\"spam.txt\"])",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": false,
+      "code_bleu": 0.47920732818470435,
+      "edit_similarity": 0.375
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "fspath(path)",
+      "got": "fspath(path)",
+      "got_raw": "fspath(path)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "f1.isatty())",
+      "got": "iotools.isatty(f1))",
+      "got_raw": "iotools.isatty(f1))",
+      "exact_match": false,
+      "code_bleu": 0.42728700639623407,
+      "edit_similarity": 0.6451612903225806
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "m1.isclosed())",
+      "got": "m1.closed)",
+      "got_raw": "m1.closed)",
+      "exact_match": false,
+      "code_bleu": 0.3387551654364098,
+      "edit_similarity": 0.8333333333333334
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "m2.isclosed())",
+      "got": "m2.isclosed())",
+      "got_raw": "m2.isclosed())",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "m1.isclosed())",
+      "got": "m1.closed)",
+      "got_raw": "m1.closed)",
+      "exact_match": false,
+      "code_bleu": 0.3387551654364098,
+      "edit_similarity": 0.8333333333333334
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "m2.isclosed())",
+      "got": "m2.isclosed())",
+      "got_raw": "m2.isclosed())",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "matcher(\"foo.py\"))",
+      "got": "matcher)",
+      "got_raw": "matcher)",
+      "exact_match": false,
+      "code_bleu": 0.23784142300054423,
+      "edit_similarity": 0.6153846153846154
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "matcher(\"foo.PY\"))",
+      "got": "matcher(\"foo.txt\"))",
+      "got_raw": "matcher(\"foo.txt\"))",
+      "exact_match": false,
+      "code_bleu": 0.45180100180492244,
+      "edit_similarity": 0.8648648648648649
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "matcher(\"FOO.py\"))",
+      "got": "not",
+      "got_raw": "not matcher(\"foo.PY\"))",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.09523809523809523
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "self.fs.exists(\"a\"))",
+      "got": "self.fs.exists(\"egg/foo.pyc\"))",
+      "got_raw": "self.fs.exists(\"egg/foo.pyc\"))",
+      "exact_match": false,
+      "code_bleu": 0.7071067811865475,
+      "edit_similarity": 0.76
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "self._contents(fs2))",
+      "got": "self._contents(fs2))",
+      "got_raw": "self._contents(fs2))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "check_readable(\"r\"))",
+      "got": "check_readable(\"r\"))",
+      "got_raw": "check_readable(\"r\"))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "check_readable(\"w\"))",
+      "got": "check_readable(\"w\"))",
+      "got_raw": "check_readable(\"w\"))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "check_readable(\"a\"))",
+      "got": "check_readable(\"a\"))",
+      "got_raw": "check_readable(\"a\"))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "check_writable(\"w\"))",
+      "got": "check_writable(\"w\"))",
+      "got_raw": "check_writable(\"w\"))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "check_writable(\"r\"))",
+      "got": "check_writable(\"r\"))",
+      "got_raw": "check_writable(\"r\"))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "src_fs.isempty(\"/\"))",
+      "got": "self.preserve_time)",
+      "got_raw": "self.preserve_time)",
+      "exact_match": false,
+      "code_bleu": 0.2272202082191573,
+      "edit_similarity": 0.2564102564102564
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "dst_fs.isdir(\"bar\"))",
+      "got": "dst_fs.exists(\"test.txt\"))",
+      "got_raw": "dst_fs.exists(\"test.txt\"))",
+      "exact_match": false,
+      "code_bleu": 0.32172944208038085,
+      "edit_similarity": 0.6086956521739131
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "fs1.getsyspath(\"/\"))",
+      "got": "fs1.getsyspath(\"$FOO\"))",
+      "got_raw": "fs1.getsyspath(\"$FOO\"))",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.8837209302325582
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "fs2.getsyspath(\"/\"))",
+      "got": "fs2.getsyspath(\"/\"))",
+      "got_raw": "fs2.getsyspath(\"/\"))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "fs.isdir(\"foo/bar\"))",
+      "got": "fs.isfile(\"foo/bar/test.txt\"))",
+      "got_raw": "fs.isfile(\"foo/bar/test.txt\"))",
+      "exact_match": false,
+      "code_bleu": 0.32172944208038085,
+      "edit_similarity": 0.72
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "ftp_fs.isfile(\"foo\"))",
+      "got": "ftp_fs.isfile(join(directory",
+      "got_raw": "ftp_fs.isfile(join(directory, \"foo\")))",
+      "exact_match": false,
+      "code_bleu": 0.5169731539571706,
+      "edit_similarity": 0.6530612244897959
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "src_fs.exists(\"foo\"))",
+      "got": "dst_fs.exists(\"test.txt\"))",
+      "got_raw": "dst_fs.exists(\"test.txt\"))",
+      "exact_match": false,
+      "code_bleu": 0.4111336169005197,
+      "edit_similarity": 0.6808510638297872
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "os.path.isdir(_path))",
+      "got": "os.path.isdir(_path))",
+      "got_raw": "os.path.isdir(_path))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "self.fs.isdir(\"foo\"))",
+      "got": "self.fs.isdir(\"foo\"))",
+      "got_raw": "self.fs.isdir(\"foo\"))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "self.fs.getinfo(\"/\"))",
+      "got": "self.fs.getinfo(\"/\"))",
+      "got_raw": "self.fs.getinfo(\"/\"))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "self.fs.exists(\"egg\"))",
+      "got": "self.fs.exists(\"foo.py\"))",
+      "got_raw": "self.fs.exists(\"foo.py\"))",
+      "exact_match": false,
+      "code_bleu": 0.7071067811865475,
+      "edit_similarity": 0.8085106382978723
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "os.path.exists(_path))",
+      "got": "os.path.exists(_path))",
+      "got_raw": "os.path.exists(_path))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "self.fs.isfile(\"foo\"))",
+      "got": "self.fs.isfile(\"foo/bar/baz/spam.txt\"))",
+      "got_raw": "self.fs.isfile(\"foo/bar/baz/spam.txt\"))",
+      "exact_match": false,
+      "code_bleu": 0.7071067811865475,
+      "edit_similarity": 0.7213114754098361
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "self.fs.delegate_fs())",
+      "got": "self.fs)",
+      "got_raw": "self.fs)",
+      "exact_match": false,
+      "code_bleu": 0.35355339059327373,
+      "edit_similarity": 0.5333333333333333
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "src.exists(\"file.txt\"))",
+      "got": "\"file.txt\"",
+      "got_raw": "\"file.txt\"",
+      "exact_match": false,
+      "code_bleu": 0.08494336535733721,
+      "edit_similarity": 0.6060606060606061
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "dst.exists(\"file.txt\"))",
+      "got": "dst.exists(\"file.txt\"))",
+      "got_raw": "dst.exists(\"file.txt\"))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "tmp.exists(\"file.txt\"))",
+      "got": "tmp.exists(\"file.txt\"))",
+      "got_raw": "tmp.exists(\"file.txt\"))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "output_file.getvalue())",
+      "got": "output_file.getvalue())",
+      "got_raw": "output_file.getvalue())",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "self.fs.getinfo(\"foo\"))",
+      "got": "self.fs.getinfo(\"foo\"))",
+      "got_raw": "self.fs.getinfo(\"foo\"))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "re.match(translated, m))",
+      "got": "re.match(translated, \"/\" + m))",
+      "got_raw": "re.match(translated, \"/\" + m))",
+      "exact_match": false,
+      "code_bleu": 0.6340466277046861,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "fs.isdir(\"foo/bar/baz\"))",
+      "got": "fs.exists(\"foo/bar/baz/egg\"))",
+      "got_raw": "fs.exists(\"foo/bar/baz/egg\"))",
+      "exact_match": false,
+      "code_bleu": 0.32172944208038085,
+      "edit_similarity": 0.7924528301886793
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "os.path.isdir(_temp_dir))",
+      "got": "os.path.exists(_temp_dir))",
+      "got_raw": "os.path.exists(_temp_dir))",
+      "exact_match": false,
+      "code_bleu": 0.6580370064762462,
+      "edit_similarity": 0.8627450980392157
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "self.cached.isdir(\"foo\"))",
+      "got": "self.cached.isdir(\"foo\"))",
+      "got_raw": "self.cached.isdir(\"foo\"))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "self.cached.isdir(\"egg\"))",
+      "got": "self.cached.isdir(\"egg\"))",
+      "got_raw": "self.cached.isdir(\"egg\"))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "self.cached.isdir(\"spam\"))",
+      "got": "self.cached.isdir(\"foo/bar\"))",
+      "got_raw": "self.cached.isdir(\"foo/bar\"))",
+      "exact_match": false,
+      "code_bleu": 0.7071067811865475,
+      "edit_similarity": 0.8363636363636363
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "self.cached.isfile(\"egg\"))",
+      "got": "self.cached.isfile(\"egg\"))",
+      "got_raw": "self.cached.isfile(\"egg\"))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "None)",
+      "got": "0.0)",
+      "got_raw": "0.0)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "PyFilesystem/pyfilesystem2",
+      "expected": "None)",
+      "got": "None)",
+      "got_raw": "None)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "None",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0.0",
+      "got_raw": "0.0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "3",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "2",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "4",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "3",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "2",
+      "got": "0.0",
+      "got_raw": "0.0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "4",
+      "got": "0.0",
+      "got_raw": "0.0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "1",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "5",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "3",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "3",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "7",
+      "got": "7",
+      "got_raw": "7",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "5",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "5",
+      "got": "0.0",
+      "got_raw": "0.0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"\"",
+      "got": "\"recall(query='hello')\"",
+      "got_raw": "\"recall(query='hello')\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.16
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"\"",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"a\"",
+      "got": "\"a\"",
+      "got_raw": "\"a\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"42\"",
+      "got": "42",
+      "got_raw": "42",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"42\"",
+      "got": "\"42\"",
+      "got_raw": "\"42\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"52\"",
+      "got": "\"10",
+      "got_raw": "\"10 * 5 + 2\"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"db\"",
+      "got": "\"schema\"",
+      "got_raw": "\"schema\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"jq\"",
+      "got": "\"jq\"",
+      "got_raw": "\"jq\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"ok\"",
+      "got": "\"ok\"",
+      "got_raw": "\"ok\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"hi\"",
+      "got": "\"hi\"",
+      "got_raw": "\"hi\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"ok\"",
+      "got": "\"ok\"",
+      "got_raw": "\"ok\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"str\"",
+      "got": "\"string\"",
+      "got_raw": "\"string\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7692307692307693
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"sig\"",
+      "got": "\"sig\"",
+      "got_raw": "\"sig\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"CEO\"",
+      "got": "\"CEO\"",
+      "got_raw": "\"CEO\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"CTO\"",
+      "got": "\"CTO\"",
+      "got_raw": "\"CTO\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"web\"",
+      "got": "\"web\"",
+      "got_raw": "\"web\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"abc\"",
+      "got": "\"abc\"",
+      "got_raw": "\"abc\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"xyz\"",
+      "got": "\"xyz\"",
+      "got_raw": "\"xyz\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"pdf\"",
+      "got": "\"pdf\"",
+      "got_raw": "\"pdf\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"rss\"",
+      "got": "\"rss\"",
+      "got_raw": "\"rss\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"web\"",
+      "got": "\"web\"",
+      "got_raw": "\"web\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"bar\"",
+      "got": "\"bar\"",
+      "got_raw": "\"bar\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"llm\"",
+      "got": "\"not_started\"",
+      "got_raw": "\"not_started\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"raw\"",
+      "got": "{\"mode\":",
+      "got_raw": "{\"mode\": \"raw\"}",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"web\"",
+      "got": "\"web\"",
+      "got_raw": "\"web\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"grok\"",
+      "got": "\"gsk\"",
+      "got_raw": "\"gsk\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7272727272727273
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"high\"",
+      "got": "\"high\"",
+      "got_raw": "\"high\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"user\"",
+      "got": "ctx.call_id",
+      "got_raw": "ctx.call_id",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"done\"",
+      "got": "\"done\"",
+      "got_raw": "\"done\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"push\"",
+      "got": "\"push\"",
+      "got_raw": "\"push\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"test\"",
+      "got": "\"test\"",
+      "got_raw": "\"test\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"test\"",
+      "got": "\"test\"",
+      "got_raw": "\"test\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"tu_1\"",
+      "got": "\"tu_1\"",
+      "got_raw": "\"tu_1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "\"user\"",
+      "got": "\"user\"",
+      "got_raw": "\"user\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "7 + 64",
+      "got": "64",
+      "got_raw": "64",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "args.func",
+      "got": "args.func.__name__",
+      "got_raw": "args.func.__name__",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "result[0]",
+      "got": "result[0]",
+      "got_raw": "result[0]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "blocks[0]",
+      "got": "blocks[0]",
+      "got_raw": "blocks[0]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "env.locals",
+      "got": "env.get_history(0)",
+      "got_raw": "env.get_history(0)",
+      "exact_match": false,
+      "code_bleu": 0.2907153684841096,
+      "edit_similarity": 0.35714285714285715
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "repl.locals",
+      "got": "repl.context",
+      "got_raw": "repl.context",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.5217391304347826
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "repl.locals",
+      "got": "repl.globals",
+      "got_raw": "repl.globals",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.8695652173913043
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "row[\"error\"]",
+      "got": "row[\"error\"]",
+      "got_raw": "row[\"error\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "result.error",
+      "got": "result.error",
+      "got_raw": "result.error",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "result.stdout",
+      "got": "result",
+      "got_raw": "result",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.631578947368421
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "result.stderr",
+      "got": "result.stderr",
+      "got_raw": "result.stderr",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "row[\"content\"]",
+      "got": "row[\"content\"]",
+      "got_raw": "row[\"content\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "ToolCategory.WEB",
+      "got": "ToolCategory.WEB",
+      "got_raw": "ToolCategory.WEB",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "ToolCategory.WEB",
+      "got": "ToolCategory.WEB",
+      "got_raw": "ToolCategory.WEB",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "s",
+      "got": "s",
+      "got_raw": "s",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "p",
+      "got": "p",
+      "got_raw": "p",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "d",
+      "got": "d",
+      "got_raw": "d",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "h",
+      "got": "h",
+      "got_raw": "h",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "v2",
+      "got": "v2",
+      "got_raw": "v2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "c2",
+      "got": "c2",
+      "got_raw": "c2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "s2",
+      "got": "s2",
+      "got_raw": "s2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "k2",
+      "got": "k2",
+      "got_raw": "k2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "ts",
+      "got": "ts",
+      "got_raw": "ts",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "ns",
+      "got": "ns",
+      "got_raw": "ns",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "cfg",
+      "got": "cfg",
+      "got_raw": "cfg",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "att",
+      "got": "att",
+      "got_raw": "att",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "mid",
+      "got": "mid",
+      "got_raw": "mid",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "id3",
+      "got": "id3",
+      "got_raw": "id3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "cfg",
+      "got": "cfg",
+      "got_raw": "cfg",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "dsn",
+      "got": "dsn",
+      "got_raw": "dsn",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "dsn",
+      "got": "dsn",
+      "got_raw": "dsn",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "cfg",
+      "got": "cfg",
+      "got_raw": "cfg",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "row",
+      "got": "row",
+      "got_raw": "row",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "sql",
+      "got": "sql",
+      "got_raw": "sql",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "url",
+      "got": "url",
+      "got_raw": "url",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "url",
+      "got": "url",
+      "got_raw": "url",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "ids",
+      "got": "ids",
+      "got_raw": "ids",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "val",
+      "got": "val",
+      "got_raw": "val",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "ids",
+      "got": "ids",
+      "got_raw": "ids",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "ids",
+      "got": "ids",
+      "got_raw": "ids",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "str(wm_id)",
+      "got": "wm_id",
+      "got_raw": "wm_id",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "float(score)",
+      "got": "score",
+      "got_raw": "score",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.5882352941176471
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "float(trust)",
+      "got": "trust",
+      "got_raw": "trust",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.5882352941176471
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "len(verifier)",
+      "got": "len(verifier)",
+      "got_raw": "len(verifier)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "QuixiAI/Hexis",
+      "expected": "float(alignment)",
+      "got": "alignment",
+      "got_raw": "alignment",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.72
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "task_state.cancel_scope._tasks",
+      "got": "self._tasks",
+      "got_raw": "self._tasks",
+      "exact_match": false,
+      "code_bleu": 0.3567621345008163,
+      "edit_similarity": 0.4878048780487805
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "handle_exc",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "RuntimeError, match=\"^foo$\")",
+      "got": "ValueError))",
+      "got_raw": "ValueError))",
+      "exact_match": false,
+      "code_bleu": 0.2259005009024612,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "TypeError, match=\"returned self\")",
+      "got": "TypeError)",
+      "got_raw": "TypeError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.46511627906976744
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "TypeError, match=\"returned a generator\")",
+      "got": "TypeError)",
+      "got_raw": "TypeError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "TypeError, match=\"returned an async generator\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.2413793103448276
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "TypeError, match=\"did not return a context manager\")",
+      "got": "TypeError)",
+      "got_raw": "TypeError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.3225806451612903
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "TypeError, match=\"returned a coroutine object instead of\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.2028985507246377
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "TypeError, match=\"did not return an async context manager\")",
+      "got": "TypeError)",
+      "got_raw": "TypeError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.2898550724637681
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "RuntimeError, match=\"^this DummyContextManager has not been entered yet$\")",
+      "got": "pytest.raises(RuntimeError)",
+      "got_raw": " pytest.raises(RuntimeError)",
+      "exact_match": false,
+      "code_bleu": 0.24446151121745047,
+      "edit_similarity": 0.25742574257425743
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "RuntimeError, match=\"^this DummyContextManager has already been entered$\")",
+      "got": "cm as cm2:",
+      "got_raw": " cm as cm2:",
+      "exact_match": false,
+      "code_bleu": 0.2014250264904512,
+      "edit_similarity": 0.14285714285714285
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "RuntimeError, match=\"^this DummyAsyncContextManager has not been entered yet$\")",
+      "got": "pytest.raises(RuntimeError)",
+      "got_raw": " pytest.raises(RuntimeError)",
+      "exact_match": false,
+      "code_bleu": 0.24446151121745047,
+      "edit_similarity": 0.24528301886792453
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "RuntimeError, match=\"^this DummyAsyncContextManager has already been entered$\")",
+      "got": "pytest.raises(RuntimeError)",
+      "got_raw": " pytest.raises(RuntimeError)",
+      "exact_match": false,
+      "code_bleu": 0.24446151121745047,
+      "edit_similarity": 0.24528301886792453
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "3",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"wait\"",
+      "got": "\"coro\"",
+      "got_raw": "\"coro\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "host_task.id",
+      "got": "host_task.id",
+      "got_raw": "host_task.id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "expected_name",
+      "got": "expected_name",
+      "got_raw": "expected_name",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"tests.test_debugging.test_main_task_name.<locals>.main\"",
+      "got": "\"main\"",
+      "got_raw": "\"main\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.1935483870967742
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "anyio.BrokenWorkerInterpreter",
+      "got": "anyio.BrokenWorkerInterpreter",
+      "got_raw": "anyio.BrokenWorkerInterpreter",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "0)",
+      "got": "0.0)",
+      "got_raw": "0.0)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "math.inf)",
+      "got": "math.inf)",
+      "got_raw": "math.inf)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "deadline - fake_current_time)",
+      "got": "500.102352)",
+      "got_raw": "500.102352)",
+      "exact_match": false,
+      "code_bleu": 0.2686424829558855,
+      "edit_similarity": 0.05
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "uri",
+      "got": "uri",
+      "got_raw": "uri",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "100",
+      "got": "100",
+      "got_raw": "100",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"c:\"",
+      "got": "\"\\\\c\"",
+      "got_raw": "\"\\\\c\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "path",
+      "got": "path",
+      "got_raw": "path",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "path2",
+      "got": "path2",
+      "got_raw": "path2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "path1",
+      "got": "path1",
+      "got_raw": "path1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "0o444",
+      "got": "0o444",
+      "got_raw": "0o444",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "0o600",
+      "got": "0o600",
+      "got_raw": "0o600",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"bar\"",
+      "got": "\"bar\"",
+      "got_raw": "\"bar\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\".zip\"",
+      "got": "\".zip\"",
+      "got_raw": "\".zip\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "7",
+      "got": "7",
+      "got_raw": "7",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "6",
+      "got": "6",
+      "got_raw": "6",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"foo\"",
+      "got": "\"foo\"",
+      "got_raw": "\"foo\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "3",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"a\"",
+      "got": "\"a\"",
+      "got_raw": "\"a\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"b\"",
+      "got": "\"b\"",
+      "got_raw": "\"b\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "index",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "i + 1",
+      "got": "i",
+      "got_raw": "i",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "cancel",
+      "got": "cancel",
+      "got_raw": "cancel",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "LookupError)",
+      "got": "RuntimeError)",
+      "got_raw": "RuntimeError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.56
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "ValueError, match=\"This token has already been used\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.34375
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "ValueError, match=\"This token does not belong to this RunVar\")",
+      "got": "pytest.raises(RuntimeError)",
+      "got_raw": " pytest.raises(RuntimeError)",
+      "exact_match": false,
+      "code_bleu": 0.22089591134157885,
+      "edit_similarity": 0.20224719101123595
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"bogus error\"",
+      "got": "\"bogus error\"",
+      "got_raw": "\"bogus error\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "result.stdout.str()",
+      "got": "\"1\"",
+      "got_raw": "\"1\"",
+      "exact_match": false,
+      "code_bleu": 0.07142857142857142,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "signal.Signals.SIGUSR1",
+      "got": "signal.SIGUSR1",
+      "got_raw": "signal.SIGUSR1",
+      "exact_match": false,
+      "code_bleu": 0.4242640687119285,
+      "edit_similarity": 0.7777777777777778
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "signal.Signals.SIGUSR2",
+      "got": "signal.Signals.SIGUSR2",
+      "got_raw": "signal.Signals.SIGUSR2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"cba\"",
+      "got": "\"cba\"",
+      "got_raw": "\"cba\"",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "str(sid)",
+      "got": "str(sid)",
+      "got_raw": "str(sid)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"ready\"",
+      "got": "\"ready\"",
+      "got_raw": "\"ready\"",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.9333333333333333
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "env[\"foo\"]",
+      "got": "\"bar\"",
+      "got_raw": "\"bar\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.26666666666666666
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "EndOfStream)",
+      "got": "ClosedResourceError)",
+      "got_raw": "ClosedResourceError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.125
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "2",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "0",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "data",
+      "got": "data",
+      "got_raw": "data",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "content",
+      "got": "content",
+      "got_raw": "content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "ValueError)",
+      "got": "IOError)",
+      "got_raw": "IOError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.631578947368421
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"helloworld\"",
+      "got": "b\"hello\"",
+      "got_raw": "b\"hello\"",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.7619047619047619
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "FileNotFoundError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5517241379310345
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "tempfile.gettempdir()",
+      "got": "tdirb",
+      "got_raw": "tdirb",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "tempfile.gettempdirb()",
+      "got": "tempfile.gettempdirb()",
+      "got_raw": "tempfile.gettempdirb()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "[\"b\", \"a\"]",
+      "got": "[\"b\", \"a\"]",
+      "got_raw": "[\"b\", \"a\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "interpreter_id_2",
+      "got": "interpreter_id_2",
+      "got_raw": "interpreter_id_2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "main_interpreter_id",
+      "got": "main_interpreter_id",
+      "got_raw": "main_interpreter_id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "ValueError, match=\"invalid literal for int\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "sys.path",
+      "got": "sys.path",
+      "got_raw": "sys.path",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "worker_pid",
+      "got": "worker_pid",
+      "got_raw": "worker_pid",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "os.getpid()",
+      "got": "os.getpid()",
+      "got_raw": "os.getpid()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "worker_pid2",
+      "got": "worker_pid2",
+      "got_raw": "worker_pid2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "real_worker",
+      "got": "real_worker",
+      "got_raw": "real_worker",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "LookupError)",
+      "got": "RuntimeError)",
+      "got_raw": "RuntimeError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.56
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "ValueError, match=\"invalid literal for int\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "[ \"b\", \"a\", ]",
+      "got": "[\"b\", \"a\"]",
+      "got_raw": "[\"b\", \"a\"]",
+      "exact_match": false,
+      "code_bleu": 0.6627256073058756,
+      "edit_similarity": 0.8695652173913043
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "3",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "6",
+      "got": "6",
+      "got_raw": "6",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "None",
+      "got": "anyio",
+      "got_raw": "anyio",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "portal2",
+      "got": "portal2",
+      "got_raw": "portal2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "no_other_refs()",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "anyio_backend_name",
+      "got": "\"anyio\"",
+      "got_raw": "\"anyio\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "expected_last_active",
+      "got": "expected_last_active",
+      "got_raw": "expected_last_active",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "KeyError, match=\"^'foo'$\")",
+      "got": "KeyError)",
+      "got_raw": "KeyError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.5142857142857142
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"fg\"",
+      "got": "b\"gh\"",
+      "got_raw": "b\"gh\"",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"ab\"",
+      "got": "b\"ab\"",
+      "got_raw": "b\"ab\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"cd\"",
+      "got": "b\"cd\"",
+      "got_raw": "b\"cd\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"abc\"",
+      "got": "b\"de\"",
+      "got_raw": "b\"de\"",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5454545454545454
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"abcd\"",
+      "got": "b\"abcd\"",
+      "got_raw": "b\"abcd\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"abcdefgh\"",
+      "got": "b\"abcd\"",
+      "got_raw": "b\"abcd\"",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.7777777777777778
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "EndOfStream)",
+      "got": "EndOfStream)",
+      "got_raw": "EndOfStream)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"xxxfooabcd\"",
+      "got": "b\"abcd\"",
+      "got_raw": "b\"abcd\"",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.7
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "IncompleteRead)",
+      "got": "IncompleteRead)",
+      "got_raw": "IncompleteRead)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "ClosedResourceError)",
+      "got": "ClosedResourceError)",
+      "got_raw": "ClosedResourceError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "2",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "5",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"lo\"",
+      "got": "b\"lo\"",
+      "got_raw": "b\"lo\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"Hel\"",
+      "got": "b\"Hel\"",
+      "got_raw": "b\"Hel\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "fileno",
+      "got": "fileno",
+      "got_raw": "fileno",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"llo\"",
+      "got": "b\"lo\"",
+      "got_raw": "b\"lo\"",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"Hello\"",
+      "got": "\"Hello\"",
+      "got_raw": "\"Hello\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "file_path",
+      "got": "file_path",
+      "got_raw": "file_path",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "EndOfStream)",
+      "got": "EndOfStream)",
+      "got_raw": "EndOfStream)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"Hello, World!\"",
+      "got": "\"Hello, World!\"",
+      "got_raw": "\"Hello, World!\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "ClosedResourceError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5806451612903226
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"item\"",
+      "got": "\"item\"",
+      "got_raw": "\"item\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"test\"",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"anyio\"",
+      "got": "\"anyio\"",
+      "got_raw": "\"anyio\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"hello\"",
+      "got": "\"anyio\"",
+      "got_raw": "\"anyio\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.42857142857142855
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"world\"",
+      "got": "\"world\"",
+      "got_raw": "\"world\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"\"",
+      "got": "b\"\"",
+      "got_raw": "b\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"hel\"",
+      "got": "b\"hel\"",
+      "got_raw": "b\"hel\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"hello\"",
+      "got": "\"hello\"",
+      "got_raw": "\"hello\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"world\"",
+      "got": "\"world\"",
+      "got_raw": "\"world\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "send_stream",
+      "got": "send_stream",
+      "got_raw": "send_stream",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"hello, world\"",
+      "got": "b\"hello, world\"",
+      "got_raw": "b\"hello, world\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "ClosedResourceError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5806451612903226
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"\u00f6\"",
+      "got": "\"\u00f6\"",
+      "got_raw": "\"\u00f6\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"\u00e5\u00e4\"",
+      "got": "\"\u00f6\"",
+      "got_raw": "\"\u00f6\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"?\"",
+      "got": "b\"?\"",
+      "got_raw": "b\"?\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"\u00e6\u00f8\"",
+      "got": "\"\u00e5\u00e4\u00f6\"",
+      "got_raw": "\"\u00e5\u00e4\u00f6\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"\ufffd\ufffd\ufffd\"",
+      "got": "\"\u00e5\u00e4\u00f6\"",
+      "got_raw": "\"\u00e5\u00e4\u00f6\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"hello\"",
+      "got": "\"hello\"",
+      "got_raw": "\"hello\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"\\xc3\\xa5\\xc3\\xa4\\xc3\\xb6\"",
+      "got": "b\"\u00e5\u00e4\u00f6\"",
+      "got_raw": "b\"\u00e5\u00e4\u00f6\"",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.18181818181818182
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "exc",
+      "got": "exc",
+      "got_raw": "exc",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"h2\"",
+      "got": "\"h2\"",
+      "got_raw": "\"h2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "False",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"olleh\"",
+      "got": "b\"olleh\"",
+      "got_raw": "b\"olleh\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "\"dummy2\"",
+      "got": "\"dummy2\"",
+      "got_raw": "\"dummy2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "agronholm/anyio",
+      "expected": "b\"hello\"",
+      "got": "b\"hello\"",
+      "got_raw": "b\"hello\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "expected_count)",
+      "got": "expected_count)",
+      "got_raw": "expected_count)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "*expected_violations_when_fixing)",
+      "got": "*expected_violations_when_fixing)",
+      "got_raw": "*expected_violations_when_fixing)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "*expected_violations_when_checking)",
+      "got": "expected_violations_when_checking)",
+      "got_raw": "expected_violations_when_checking)",
+      "exact_match": false,
+      "code_bleu": 0.4714045207910317,
+      "edit_similarity": 0.9855072463768116
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "ALL_TAGS",
+      "got": "ALL_TAGS",
+      "got_raw": "ALL_TAGS",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "*name_contents)",
+      "got": "name_contents)",
+      "got_raw": "name_contents)",
+      "exact_match": false,
+      "code_bleu": 0.4714045207910317,
+      "edit_similarity": 0.9655172413793104
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "expected_pylintrc)",
+      "got": "=",
+      "got_raw": " = project_default",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "datadir / \"real.yaml\")",
+      "got": "uss(datadir / \"real.toml\")",
+      "got_raw": "uss(datadir / \"real.toml\")",
+      "exact_match": false,
+      "code_bleu": 0.31239399369202553,
+      "edit_similarity": 0.8333333333333334
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "0)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "1)",
+      "got": "3)",
+      "got_raw": "3)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "3)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "4)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "5)",
+      "got": "5)",
+      "got_raw": "5)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "2)",
+      "got": "2)",
+      "got_raw": "2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "(expected_enum, expected_timedelta)",
+      "got": "(expected_enum, expected_timedelta)",
+      "got_raw": "(expected_enum, expected_timedelta)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "unchanged)",
+      "got": "(tmp_path)",
+      "got_raw": "(tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "f\"\"\" [{CONFIG_TOOL_NITPICK_KEY}] style = [ \"my-style.toml\",] \"\"\")",
+      "got": "_toml = project.toml",
+      "got_raw": "_toml = project.toml",
+      "exact_match": false,
+      "code_bleu": 0.16332005089231103,
+      "edit_similarity": 0.2823529411764706
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "f\"\"\" [{CONFIG_TOOL_NITPICK_KEY}]",
+      "got": "_config = project.config",
+      "got_raw": "_config = project.config",
+      "exact_match": false,
+      "code_bleu": 0.14348430291633957,
+      "edit_similarity": 0.03571428571428571
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "f\"\"\" {before}[{CONFIG_TOOL_NITPICK_KEY}]{after}",
+      "got": "_config = project.config",
+      "got_raw": "_config = project.config",
+      "exact_match": false,
+      "code_bleu": 0.08199103023790832,
+      "edit_similarity": 0.08450704225352113
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "f\"\"\" [{CONFIG_TOOL_NITPICK_KEY}]",
+      "got": "_toml = project.toml",
+      "got_raw": "_toml = project.toml",
+      "exact_match": false,
+      "code_bleu": 0.14348430291633957,
+      "edit_similarity": 0.038461538461538464
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "path",
+      "got": "path",
+      "got_raw": "path",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "set()",
+      "got": "set()",
+      "got_raw": "set()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "message.strip().casefold()",
+      "got": "message.casefold()",
+      "got_raw": "message.casefold()",
+      "exact_match": false,
+      "code_bleu": 0.4671646751409525,
+      "edit_similarity": 0.8181818181818182
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "{ some_directory / \"README.md\", }",
+      "got": "{some_directory / \"README.md\"}",
+      "got_raw": "{some_directory / \"README.md\"}",
+      "exact_match": false,
+      "code_bleu": 0.6627256073058756,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "{ some_directory / \"src\" / \"module.py\", }",
+      "got": "{some_directory / \"src\" / \"module.py\"}",
+      "got_raw": "{some_directory / \"src\" / \"module.py\"}",
+      "exact_match": false,
+      "code_bleu": 0.760761525903735,
+      "edit_similarity": 0.9620253164556962
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "{ some_directory / \"file1.txt\", some_directory / \"src\" / \"module.py\", }",
+      "got": "{some_directory / \"README.md\"}",
+      "got_raw": "{some_directory / \"README.md\"}",
+      "exact_match": false,
+      "code_bleu": 0.22386873579657127,
+      "edit_similarity": 0.4752475247524752
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "{ some_directory / GIT_IGNORE, some_directory / \"README.md\", some_directory / \"src\" / \"module.py\", }",
+      "got": "{some_directory / \"README.md\"}",
+      "got_raw": "{some_directory / \"README.md\"}",
+      "exact_match": false,
+      "code_bleu": 0.20898134530513188,
+      "edit_similarity": 0.46153846153846156
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "{ some_directory / \"file1.txt\", some_directory / \"README.md\", some_directory / \"src\" / \"module.py\", }",
+      "got": "{some_directory / \"README.md\"}",
+      "got_raw": "{some_directory / \"README.md\"}",
+      "exact_match": false,
+      "code_bleu": 0.20898134530513188,
+      "edit_similarity": 0.4580152671755725
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "original_file)",
+      "got": "EDITOR_CONFIG)",
+      "got_raw": "EDITOR_CONFIG)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.14285714285714285
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "expected_generic_ini)",
+      "got": "your-section\"",
+      "got_raw": "your-section\", expected_generic_ini)",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.29411764705882354
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "expected_editor_config)",
+      "got": "\".\")",
+      "got_raw": "\".\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.07407407407407407
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "datadir / \"3-expected-setup.cfg\")",
+      "got": "add_file(datadir / \"ini_multiline_comment.ini\")",
+      "got_raw": "add_file(datadir / \"ini_multiline_comment.ini\")",
+      "exact_match": false,
+      "code_bleu": 0.31239399369202553,
+      "edit_similarity": 0.375
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "datadir / \"2-expected-editorconfig.ini\")",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"\"\" [flake8] max-line-length = 100 max-complexity = 10 \"\"\")",
+      "got": "= ProjectMock(tmp_path)",
+      "got_raw": " = ProjectMock(tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.2730120862709067,
+      "edit_similarity": 0.17073170731707318
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"\"\" [food] eat = spam,eggs,cheese,ham,salt drink = wine , bier , water \"\"\")",
+      "got": "= ProjectMock(tmp_path)",
+      "got_raw": " = ProjectMock(tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.2730120862709067,
+      "edit_similarity": 0.12244897959183673
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"\"\" [mypy] ignore_missing_imports = true [flake8] max-line-length = 120 [isort] line_length = 120 \"\"\")",
+      "got": "= ProjectMock(tmp_path)",
+      "got_raw": " = ProjectMock(tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.2730120862709067,
+      "edit_similarity": 0.16
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "Fuss( True, PYTHON_SETUP_CFG, 324, \": section [flake8] has some missing key/value pairs. Use this:\", \"\"\" [flake8] new = value \"\"\", ))",
+      "got": "= ProjectMock(datadir)",
+      "got_raw": " = ProjectMock(datadir)",
+      "exact_match": false,
+      "code_bleu": 0.094417981167399,
+      "edit_similarity": 0.15483870967741936
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "Fuss( False, PYTHON_SETUP_CFG, Violations.PARSING_ERROR.code, \": parsing error (ParsingError): Source contains parsing errors: 'simulating a captured error'\", ))",
+      "got": "= ProjectMock(tmp_path)",
+      "got_raw": " = ProjectMock(tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.094417981167399,
+      "edit_similarity": 0.11956521739130435
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"\"\" [mypy]",
+      "got": "= ProjectMock(tmp_path)",
+      "got_raw": " = ProjectMock(tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.2295748846661433,
+      "edit_similarity": 0.18181818181818182
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "Fuss( False, PYTHON_SETUP_CFG, Violations.PARSING_ERROR.code, f\": parsing error (DuplicateOptionError): While reading from {project.path_for(PYTHON_SETUP_CFG)!r} \" f\"[line 3]: option 'easy' in section 'abc' already exists\", ))",
+      "got": "_file = project.get_project_file()",
+      "got_raw": "_file = project.get_project_file()",
+      "exact_match": false,
+      "code_bleu": 0.08138270417847641,
+      "edit_similarity": 0.06923076923076923
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "Fuss( False, PYTHON_SETUP_CFG, 321, \" was not found. Create it with this content:\", \"\"\" [flake8] max-line-length = 85 max-complexity = 12 ignore = D100,D101,D102,D103,D104,D105,D106,D107,D202,E203,W503 select = E241,C,E,F,W,B,B9 \"\"\", ))",
+      "got": "= ProjectMock(tmp_path)",
+      "got_raw": " = ProjectMock(tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.094417981167399,
+      "edit_similarity": 0.06177606177606178
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "expected_package_json)",
+      "got": "(\"package.json\")",
+      "got_raw": "(\"package.json\")",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.631578947368421
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "datadir / \"3-expected.json\")",
+      "got": "_path)",
+      "got_raw": "_path)",
+      "exact_match": false,
+      "code_bleu": 0.2686424829558855,
+      "edit_similarity": 0.17647058823529413
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "err",
+      "got": "err",
+      "got_raw": "err",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "retv",
+      "got": "retv",
+      "got_raw": "retv",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "False",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"something\"",
+      "got": "[\"NITPICK_SOME_OPTION=something\"]",
+      "got_raw": "[\"NITPICK_SOME_OPTION=something\"]",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "SystemExit)",
+      "got": "SystemExit)",
+      "got_raw": "SystemExit)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"NITPICK_SOME_OPTION\"",
+      "got": "\"SOME_OPTION=something\"",
+      "got_raw": "\"SOME_OPTION=something\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5909090909090909
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"--nitpick-multi-word\"",
+      "got": "\"--multi-word\"",
+      "got_raw": "\"--multi-word\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7777777777777778
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "app2",
+      "got": "app2",
+      "got_raw": "app2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "root",
+      "got": "root",
+      "got_raw": "root",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "error)",
+      "got": "error)",
+      "got_raw": "error)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "str(err)",
+      "got": "str(err.value)",
+      "got_raw": "str(err.value)",
+      "exact_match": false,
+      "code_bleu": 0.4347208719449915,
+      "edit_similarity": 0.7272727272727273
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "apps_dir",
+      "got": "apps_dir",
+      "got_raw": "apps_dir",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "TypeError)",
+      "got": "QuitComplainingError)",
+      "got_raw": "QuitComplainingError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.45161290322580644
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "caplog.messages",
+      "got": "caplog.text",
+      "got_raw": "caplog.text",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.6153846153846154
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "QuitComplainingError)",
+      "got": "QuitComplainingError)",
+      "got_raw": "QuitComplainingError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "apps_dir / PYTHON_MANAGE_PY",
+      "got": "tmp_path",
+      "got_raw": "tmp_path",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.11428571428571428
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "f\"NIP102 No Python file was found on the root dir and subdir of {str(project.root_dir)!r}\")",
+      "got": "= Nitpick(project)",
+      "got_raw": " = Nitpick(project)",
+      "exact_match": false,
+      "code_bleu": 0.2730120862709067,
+      "edit_similarity": 0.25688073394495414
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "2)",
+      "got": "=",
+      "got_raw": " = ProjectMock(tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "1)",
+      "got": "=",
+      "got_raw": " = project.with_pyproject_toml(tomlstring(project))",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "None)",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"nitpick\"",
+      "got": "\"nitpick\"",
+      "got_raw": "\"nitpick\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "import_path",
+      "got": "import_path",
+      "got_raw": "import_path",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"andreoliwa\"",
+      "got": "\"andreoliwa\"",
+      "got_raw": "\"andreoliwa\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "git_reference",
+      "got": "git_reference",
+      "got_raw": "git_reference",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "resource_name",
+      "got": "resource_name",
+      "got_raw": "resource_name",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "RuntimeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "furl(expected_url)",
+      "got": "expected_url",
+      "got_raw": "expected_url",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "f\"token {file_token}\"",
+      "got": "f\"token={file_token}\"",
+      "got_raw": "f\"token={file_token}\"",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "expected_content_path",
+      "got": "expected_content_path",
+      "got_raw": "expected_content_path",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "mocked_version == \"0.5.3\")",
+      "got": "tmp_path)",
+      "got_raw": "tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.2149139863647084,
+      "edit_similarity": 0.17142857142857143
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "parsed.url.query.params.items()",
+      "got": "parsed.query",
+      "got_raw": "parsed.query",
+      "exact_match": false,
+      "code_bleu": 0.1928473039599675,
+      "edit_similarity": 0.5581395348837209
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "1)",
+      "got": "=",
+      "got_raw": " = ProjectMock(tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "expected_toml)",
+      "got": "_path=tmp_path)",
+      "got_raw": "_path=tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.35930411196308426,
+      "edit_similarity": 0.27586206896551724
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"\"\" [section]",
+      "got": "= ProjectMock(tmp_path)",
+      "got_raw": " = ProjectMock(tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.2295748846661433,
+      "edit_similarity": 0.2777777777777778
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"\"\" [something] x = 1",
+      "got": "= ProjectMock(tmp_path)",
+      "got_raw": " = ProjectMock(tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.17063255391931667,
+      "edit_similarity": 0.09090909090909091
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "content)",
+      "got": "content)",
+      "got_raw": "content)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "from_path",
+      "got": "from_path",
+      "got_raw": "from_path",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "dedent(after)",
+      "got": "dedent(after)",
+      "got_raw": "dedent(after)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"\"\" [my.table]",
+      "got": "\"my.table\\n\"",
+      "got_raw": "\"my.table\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.07142857142857142,
+      "edit_similarity": 0.6923076923076923
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"\"\" [my.table]",
+      "got": "\"my.table = [1",
+      "got_raw": "\"my.table = [1, 2, 3]\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.38260294162784475,
+      "edit_similarity": 0.6428571428571429
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"\"\" [my.table]",
+      "got": "\"my.table\\n\"",
+      "got_raw": "\"my.table\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.07142857142857142,
+      "edit_similarity": 0.6923076923076923
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"\"\" [my.table] key-before = 3",
+      "got": "\"my.table = {simple-key = 1}\"",
+      "got_raw": "\"my.table = {simple-key = 1}\"",
+      "exact_match": false,
+      "code_bleu": 0.041666666666666664,
+      "edit_similarity": 0.41379310344827586
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"\"\" [my.table]",
+      "got": "\"my.table\\n\"",
+      "got_raw": "\"my.table\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.07142857142857142,
+      "edit_similarity": 0.6923076923076923
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"\"\" [my.table]",
+      "got": "\"my.table = [1",
+      "got_raw": "\"my.table = [1, 2, 3]\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.38260294162784475,
+      "edit_similarity": 0.6428571428571429
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"\"\" [my.table]",
+      "got": "\"my.table\\n\"",
+      "got_raw": "\"my.table\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.07142857142857142,
+      "edit_similarity": 0.6923076923076923
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"\"\" [my.table]",
+      "got": "\"my.table\\n\"",
+      "got_raw": "\"my.table\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.07142857142857142,
+      "edit_similarity": 0.6923076923076923
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "0",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "expected_yaml)",
+      "got": "ir",
+      "got_raw": "ir / \"style.yaml\")",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "datadir / \"existing-expected.yaml\")",
+      "got": "-style.yaml\")",
+      "got_raw": "-style.yaml\")",
+      "exact_match": false,
+      "code_bleu": 0.22089591134157885,
+      "edit_similarity": 0.4166666666666667
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "datadir / \"list-by-hash-expected.yaml\")",
+      "got": "obj-hash.yaml\")",
+      "got_raw": "obj-hash.yaml\")",
+      "exact_match": false,
+      "code_bleu": 0.18575057999133598,
+      "edit_similarity": 0.48148148148148145
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "datadir / \"jmes-list-key-expected.yaml\")",
+      "got": "st-key.yaml\")",
+      "got_raw": "st-key.yaml\")",
+      "exact_match": false,
+      "code_bleu": 0.18575057999133598,
+      "edit_similarity": 0.49056603773584906
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "datadir / \"any-order.yaml\")",
+      "got": ".yaml\")",
+      "got_raw": ".yaml\")",
+      "exact_match": false,
+      "code_bleu": 0.35930411196308426,
+      "edit_similarity": 0.4117647058823529
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "datadir / \"wildcard-expected.yaml\")",
+      "got": "ppath=tmp_path)",
+      "got_raw": "ppath=tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.35930411196308426,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "datadir / \"same-key-expected.yaml\")",
+      "got": "ppath=tmp_path)",
+      "got_raw": "ppath=tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.35930411196308426,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "datadir / \"dict-search-by-key-expected.yaml\")",
+      "got": "ppath=tmp_path)",
+      "got_raw": "ppath=tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.35930411196308426,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "datadir / \"scalar-add-elements-that-do-not-exist-expected.yaml\")",
+      "got": "ppath=tmp_path)",
+      "got_raw": "ppath=tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.35930411196308426,
+      "edit_similarity": 0.1518987341772152
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "datadir / \"2-untouched-pre-commit.yaml\")",
+      "got": "\".pre-commit-config.yaml\"",
+      "got_raw": "\".pre-commit-config.yaml\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.5230769230769231
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"\"\" repos: [] \"\"\")",
+      "got": "= ProjectMock(datadir)",
+      "got_raw": " = ProjectMock(datadir)",
+      "exact_match": false,
+      "code_bleu": 0.2730120862709067,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "\"\"\" repos: - hooks: - id: whatever - grepo: glocal \"\"\")",
+      "got": "= ProjectMock(tmp_path)",
+      "got_raw": " = ProjectMock(tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.2730120862709067,
+      "edit_similarity": 0.20512820512820512
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "r\"\"\" repos: - repo: another hooks: - id: isort - repo: another hooks: \"- name: isort\\n entry: isort -sp setup.cfg\\n\" \"\"\")",
+      "got": "= ProjectMock(tmp_path)",
+      "got_raw": " = ProjectMock(tmp_path)",
+      "exact_match": false,
+      "code_bleu": 0.06204820142520606,
+      "edit_similarity": 0.125
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "datadir / \"hook-args-add.yaml\")",
+      "got": "uss(Fuss(\"Missing key/value pairs\"",
+      "got_raw": "uss(Fuss(\"Missing key/value pairs\", \"repo\"))",
+      "exact_match": false,
+      "code_bleu": 0.2295748846661433,
+      "edit_similarity": 0.18461538461538463
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "datadir / \"hook-args-change.yaml\")",
+      "got": "uss(Fuss(\"different key/value pairs\"",
+      "got_raw": "uss(Fuss(\"different key/value pairs\", \"different key/value pairs\"))",
+      "exact_match": false,
+      "code_bleu": 0.2295748846661433,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "datadir / \"uk-empty-expected.yaml\")",
+      "got": "uss(\"fuss.yaml\"",
+      "got_raw": "uss(\"fuss.yaml\", \"\"\"\\",
+      "exact_match": false,
+      "code_bleu": 0.2849383821193472,
+      "edit_similarity": 0.28
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "datadir / \"uk-default-expected.yaml\")",
+      "got": "uss(Fuss(\"repo\"",
+      "got_raw": "uss(Fuss(\"repo\", \"repo\", \"repo\"))",
+      "exact_match": false,
+      "code_bleu": 0.2295748846661433,
+      "edit_similarity": 0.11538461538461539
+    },
+    {
+      "repo": "andreoliwa/nitpick",
+      "expected": "datadir / \"uk-override-expected.yaml\")",
+      "got": "uss(Fuss(\"foo\"",
+      "got_raw": "uss(Fuss(\"foo\", \"bar\", \"baz\"))",
+      "exact_match": false,
+      "code_bleu": 0.2295748846661433,
+      "edit_similarity": 0.07692307692307693
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "2",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "6",
+      "got": "6",
+      "got_raw": "6",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "5)",
+      "got": "5)",
+      "got_raw": "5)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "2)",
+      "got": "2)",
+      "got_raw": "2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "10",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "20",
+      "got": "20",
+      "got_raw": "20",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "30",
+      "got": "30",
+      "got_raw": "30",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "10",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "20",
+      "got": "20",
+      "got_raw": "20",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "30",
+      "got": "30",
+      "got_raw": "30",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "10",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "20",
+      "got": "20",
+      "got_raw": "20",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "30",
+      "got": "30",
+      "got_raw": "30",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "10",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "20",
+      "got": "20",
+      "got_raw": "20",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "30",
+      "got": "30",
+      "got_raw": "30",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "10",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "20",
+      "got": "20",
+      "got_raw": "20",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "30",
+      "got": "30",
+      "got_raw": "30",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "2)",
+      "got": "True)",
+      "got_raw": "True)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "2)",
+      "got": "2)",
+      "got_raw": "2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "0.9",
+      "got": "0.9",
+      "got_raw": "0.9",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "0.5",
+      "got": "0.5",
+      "got_raw": "0.5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "0.0",
+      "got": "0.0",
+      "got_raw": "0.0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "0.5",
+      "got": "0.5",
+      "got_raw": "0.5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "0.9",
+      "got": "0.9",
+      "got_raw": "0.9",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "10)",
+      "got": "10)",
+      "got_raw": "10)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "15)",
+      "got": "15)",
+      "got_raw": "15)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "20)",
+      "got": "20)",
+      "got_raw": "20)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "30)",
+      "got": "30)",
+      "got_raw": "30)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "0.7",
+      "got": "0.7",
+      "got_raw": "0.7",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"en\"",
+      "got": "\"en\"",
+      "got_raw": "\"en\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"en\"",
+      "got": "\"en\"",
+      "got_raw": "\"en\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"en\"",
+      "got": "\"en\"",
+      "got_raw": "\"en\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"en\"",
+      "got": "\"en\"",
+      "got_raw": "\"en\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"en\"",
+      "got": "\"en\"",
+      "got_raw": "\"en\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"72\")",
+      "got": "\"72\")",
+      "got_raw": "\"72\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"warn\"",
+      "got": "\"warn\"",
+      "got_raw": "\"warn\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"Test\"",
+      "got": "\"Test\"",
+      "got_raw": "\"Test\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"fast\"",
+      "got": "{\"content\":",
+      "got_raw": "{\"content\": [{\"type\": \"text\", \"text\": \"fast\"}]}",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.35294117647058826
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"slow\"",
+      "got": "\"slow\"",
+      "got_raw": "\"slow\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"json\"",
+      "got": "\"json\"",
+      "got_raw": "\"json\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"user\"",
+      "got": "\"USER\"",
+      "got_raw": "\"USER\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"stop\"",
+      "got": "\"stop_sequence\"",
+      "got_raw": "\"stop_sequence\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"true\"",
+      "got": "\"10\"",
+      "got_raw": "\"10\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"Test\"",
+      "got": "\"Test\"",
+      "got_raw": "\"Test\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"tool1\"",
+      "got": "\"tool1\"",
+      "got_raw": "\"tool1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"tool2\"",
+      "got": "\"tool2\"",
+      "got_raw": "\"tool2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"stop\")",
+      "got": "\"end_turn\")",
+      "got_raw": "\"end_turn\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"user\")",
+      "got": "\"tool\")",
+      "got_raw": "\"tool\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.42857142857142855
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"en-US\"",
+      "got": "\"en-US\"",
+      "got_raw": "\"en-US\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"user\")",
+      "got": "\"user\")",
+      "got_raw": "\"user\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"stop\")",
+      "got": "\"complete\")",
+      "got_raw": "\"complete\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5555555555555556
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"en-US\"",
+      "got": "\"en-US\"",
+      "got_raw": "\"en-US\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"user\")",
+      "got": "\"assistant\")",
+      "got_raw": "\"assistant\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.42105263157894735
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"stop\")",
+      "got": "\"stop\")",
+      "got_raw": "\"stop\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"model\"",
+      "got": "\"assistant\"",
+      "got_raw": "\"assistant\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"Hello\"",
+      "got": "\"Hello\"",
+      "got_raw": "\"Hello\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"model\")",
+      "got": "\"assistant\")",
+      "got_raw": "\"assistant\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.3
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"Hello!\"",
+      "got": "\"Hello!\"",
+      "got_raw": "\"Hello!\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\" Hello\"",
+      "got": "\"Hello\"",
+      "got_raw": "\"Hello\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9333333333333333
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"array\")",
+      "got": "\"array\")",
+      "got_raw": "\"array\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"array\")",
+      "got": "\"List[dict]\")",
+      "got_raw": "\"List[dict]\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"email\")",
+      "got": "\"email\")",
+      "got_raw": "\"email\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"object\")",
+      "got": "\"object\")",
+      "got_raw": "\"object\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"object\")",
+      "got": "\"object\")",
+      "got_raw": "\"object\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"string\")",
+      "got": "\"string\")",
+      "got_raw": "\"string\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"tool123\")",
+      "got": "\"get_weather\"",
+      "got_raw": "\"get_weather\", normalized_response.choices[0].message.tool_calls[0].name)",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.2608695652173913
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"tool123\")",
+      "got": "\"tool123\")",
+      "got_raw": "\"tool123\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"function\"",
+      "got": "\"function\"",
+      "got_raw": "\"function\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"success\")",
+      "got": "\"success\")",
+      "got_raw": "\"success\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"Celsius\")",
+      "got": "\"Celsius\")",
+      "got_raw": "\"Celsius\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"read_file\"",
+      "got": "\"read_file\"",
+      "got_raw": "\"read_file\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"test_tool\"",
+      "got": "\"test_tool\"",
+      "got_raw": "\"test_tool\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"top_song\")",
+      "got": "\"top_song\")",
+      "got_raw": "\"top_song\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"function\")",
+      "got": "\"function\")",
+      "got_raw": "\"function\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"audio/wav\"",
+      "got": "\"audio/wav\"",
+      "got_raw": "\"audio/wav\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"SSE result\"",
+      "got": "\"SSE result\"",
+      "got_raw": "\"SSE result\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"assistant\")",
+      "got": "\"assistant\")",
+      "got_raw": "\"assistant\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"assistant\")",
+      "got": "\"assistant\")",
+      "got_raw": "\"assistant\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"Hello world\"",
+      "got": "\"Hello world\"",
+      "got_raw": "\"Hello world\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"test-server\"",
+      "got": "\"test-server\"",
+      "got_raw": "\"test-server\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"tool_calls\")",
+      "got": "\"tool_use\")",
+      "got_raw": "\"tool_use\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"tool_calls\")",
+      "got": "\"function_call\")",
+      "got_raw": "\"function_call\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6896551724137931
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"Hello world\"",
+      "got": "\"Hello world\"",
+      "got_raw": "\"Hello world\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"final result\"",
+      "got": "{\"content\": [{\"type\": \"text\"",
+      "got_raw": "{\"content\": [{\"type\": \"text\", \"text\": \"final result\"}]}",
+      "exact_match": false,
+      "code_bleu": 0.13485111859503685,
+      "edit_similarity": 0.23809523809523808
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"get_weather\")",
+      "got": "\"get_weather\")",
+      "got_raw": "\"get_weather\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"verbose_json\"",
+      "got": "\"verbose_json\"",
+      "got_raw": "\"verbose_json\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"get_weather\")",
+      "got": "\"get_weather\")",
+      "got_raw": "\"get_weather\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"test-api-key\"",
+      "got": "\"test-api-key\"",
+      "got_raw": "\"test-api-key\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"get_weather\")",
+      "got": "\"get_weather\")",
+      "got_raw": "\"get_weather\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"test_simple\")",
+      "got": "\"test_simple\")",
+      "got_raw": "\"test_simple\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"Hello, World!\"",
+      "got": "\"Hello, World!\"",
+      "got_raw": "\"Hello, World!\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"test-hf-token\"",
+      "got": "\"test-hf-token\"",
+      "got_raw": "\"test-hf-token\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"San Francisco\")",
+      "got": "\"San Francisco\")",
+      "got_raw": "\"San Francisco\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"Mixed tools work!\"",
+      "got": "\"Mixed tools work!\"",
+      "got_raw": "\"Mixed tools work!\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"Get the weather.\")",
+      "got": "\"Get the weather.\")",
+      "got_raw": "\"Get the weather.\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "'{\"sign\": \"WZPZ\"}')",
+      "got": "{\"sign\": \"WZPZ\"})",
+      "got_raw": "{\"sign\": \"WZPZ\"})",
+      "exact_match": false,
+      "code_bleu": 0.22089591134157885,
+      "edit_similarity": 0.9444444444444444
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"Test error message\"",
+      "got": "\"Test error message\"",
+      "got_raw": "\"Test error message\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "\"get_exchange_rate\")",
+      "got": "[0].function_call)",
+      "got_raw": "[0].function_call)",
+      "exact_match": false,
+      "code_bleu": 0.22089591134157885,
+      "edit_similarity": 0.15789473684210525
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "tool",
+      "got": "tool",
+      "got_raw": "tool",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "result",
+      "got": "result",
+      "got_raw": "result",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "result",
+      "got": "result",
+      "got_raw": "result",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "content",
+      "got": "content",
+      "got_raw": "content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "headers",
+      "got": "headers",
+      "got_raw": "headers",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "ASRError)",
+      "got": "ValueError",
+      "got_raw": "ValueError, match=\"Original error\")",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.5263157894736842
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "TypeError)",
+      "got": "TypeError)",
+      "got_raw": "TypeError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "Exception)",
+      "got": "Exception)",
+      "got_raw": "Exception)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "tool_names",
+      "got": "tool_names",
+      "got_raw": "tool_names",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "tool_names",
+      "got": "tool_names",
+      "got_raw": "tool_names",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "audio_data",
+      "got": "audio_data",
+      "got_raw": "audio_data",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "TypeError)",
+      "got": "TypeError)",
+      "got_raw": "TypeError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "call_kwargs",
+      "got": "call_kwargs",
+      "got_raw": "call_kwargs",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "tool_names)",
+      "got": "tool_names)",
+      "got_raw": "tool_names)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "schema_json)",
+      "got": "schema_json)",
+      "got_raw": "schema_json)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "schema_copy)",
+      "got": "schema_copy)",
+      "got_raw": "schema_copy)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "text_content)",
+      "got": "text_content)",
+      "got_raw": "text_content)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "input_schema)",
+      "got": "input_schema)",
+      "got_raw": "input_schema)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "original_error",
+      "got": "original_error",
+      "got_raw": "original_error",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "selected_model",
+      "got": "selected_model",
+      "got_raw": "selected_model",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "PROVIDER_PARAMS",
+      "got": "PROVIDER_PARAMS",
+      "got_raw": "PROVIDER_PARAMS",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "expected_tool_spec",
+      "got": "expected_tool_spec",
+      "got_raw": "expected_tool_spec",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "GOOGLE_LANGUAGE_MAP",
+      "got": "GOOGLE_LANGUAGE_MAP",
+      "got_raw": "GOOGLE_LANGUAGE_MAP",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "response_text_content",
+      "got": "response_text_content",
+      "got_raw": "response_text_content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "response_text_content",
+      "got": "response_text_content",
+      "got_raw": "response_text_content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "response_text_content",
+      "got": "response_text_content",
+      "got_raw": "response_text_content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "response_text_content",
+      "got": "response_text_content",
+      "got_raw": "response_text_content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "expected_content_type",
+      "got": "\"audio\"",
+      "got_raw": "\"audio\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.14285714285714285
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "response_text_content",
+      "got": "response_text_content",
+      "got_raw": "response_text_content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "response_text_content",
+      "got": "response_text_content",
+      "got_raw": "response_text_content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "response_text_content",
+      "got": "response_text_content",
+      "got_raw": "response_text_content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "response_text_content",
+      "got": "response_text_content",
+      "got_raw": "response_text_content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "results[0])",
+      "got": "results[0])",
+      "got_raw": "results[0])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "provider.api_key",
+      "got": "provider.api_key",
+      "got_raw": "provider.api_key",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "call_args.args[0]",
+      "got": "call_args[0][1][\"model\"]",
+      "got_raw": "call_args[0][1][\"model\"]",
+      "exact_match": false,
+      "code_bleu": 0.24808415001701817,
+      "edit_similarity": 0.5853658536585366
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "provider.project_id",
+      "got": "\"test-project-id\"",
+      "got_raw": "\"test-project-id\"",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.5555555555555556
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "provider.service_url",
+      "got": "provider.service_url",
+      "got_raw": "provider.service_url",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "COMMON_PARAMS[\"language\"]",
+      "got": "COMMON_PARAMS",
+      "got_raw": "COMMON_PARAMS",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.6842105263157895
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "ValueError, match=\"Test error\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.5238095238095238
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "ImportError, match=\"mcp.*package\")",
+      "got": "ValueError, match=\"MCP package not installed\")",
+      "got_raw": "ValueError, match=\"MCP package not installed\")",
+      "exact_match": false,
+      "code_bleu": 0.4347208719449915,
+      "edit_similarity": 0.575
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "ASRError, match=\"Specific ASR error\")",
+      "got": "ASRError)",
+      "got_raw": "ASRError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.391304347826087
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "ValueError, match=\"must have 'name'\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.4583333333333333
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "response.choices[0].message.tool_calls)",
+      "got": "response.error)",
+      "got_raw": "response.error)",
+      "exact_match": false,
+      "code_bleu": 0.18477554478349628,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "[])",
+      "got": "\"User:",
+      "got_raw": "\"User: Hello, how are you?\")",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "[])",
+      "got": "\"User:",
+      "got_raw": "\"User: What is the most popular song on WZPZ?\")",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "{})",
+      "got": "{})",
+      "got_raw": "{})",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "[])",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "{\"diarize\": True}",
+      "got": "{\"diarize\": True}",
+      "got_raw": "{\"diarize\": True}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "{\"language\": \"en\"}",
+      "got": "{\"language\": \"en\"}",
+      "got_raw": "{\"language\": \"en\"}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "{\"language\": None}",
+      "got": "{\"language\": \"en\"}",
+      "got_raw": "{\"language\": \"en\"}",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.8333333333333334
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "{\"punctuate\": True}",
+      "got": "{\"punctuate\": True}",
+      "got_raw": "{\"punctuate\": True}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "[\"technical terms\"]",
+      "got": "[\"technical terms\"]",
+      "got_raw": "[\"technical terms\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "[expected_message])",
+      "got": "[expected_message])",
+      "got_raw": "[expected_message])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "[\"required_param\"])",
+      "got": "[\"required_param\"])",
+      "got_raw": "[\"required_param\"])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "{\"temperature\": 0.5}",
+      "got": "{\"temperature\": 0.5}",
+      "got_raw": "{\"temperature\": 0.5}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "{\"unknown_param\": True}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.23784142300054423,
+      "edit_similarity": 0.16
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "{\"keywords\": [\"meeting\"]}",
+      "got": "[\"meeting\"]",
+      "got_raw": "[\"meeting\"]",
+      "exact_match": false,
+      "code_bleu": 0.36038417796587763,
+      "edit_similarity": 0.6111111111111112
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "{\"temperature\": \"15\", \"unit\": \"Celsius\"})",
+      "got": "\"15\"",
+      "got_raw": "\"15\"",
+      "exact_match": false,
+      "code_bleu": 0.05946035575013606,
+      "edit_similarity": 0.17777777777777778
+    },
+    {
+      "repo": "andrewyng/aisuite",
+      "expected": "str(context.exception))",
+      "got": "\"Invalid",
+      "got_raw": "\"Invalid type for location\")",
+      "exact_match": false,
+      "code_bleu": 0.1290860005156921,
+      "edit_similarity": 0.12903225806451613
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "200",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "422",
+      "got": "422",
+      "got_raw": "422",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "2",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "True",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"192.168.1.1\"",
+      "got": "\"192.168.1.1\"",
+      "got_raw": "\"192.168.1.1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "custom_session",
+      "got": "custom_session",
+      "got_raw": "custom_session",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"https://api.ipify.org\")",
+      "got": "_session.get.assert_called_once()",
+      "got_raw": "_session.get.assert_called_once()",
+      "exact_match": false,
+      "code_bleu": 0.18575057999133598,
+      "edit_similarity": 0.17543859649122806
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"http://ipecho.net/plain\")",
+      "got": "_get.assert_called_once()",
+      "got_raw": "_get.assert_called_once()",
+      "exact_match": false,
+      "code_bleu": 0.2730120862709067,
+      "edit_similarity": 0.19607843137254902
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"mocked-ip-response\"",
+      "got": "proxy[\"ip\"]",
+      "got_raw": "proxy[\"ip\"]",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.25806451612903225
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"https://api.ipify.org\")",
+      "got": "_session.get.assert_called_once()",
+      "got_raw": "_session.get.assert_called_once()",
+      "exact_match": false,
+      "code_bleu": 0.18575057999133598,
+      "edit_similarity": 0.17543859649122806
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"http://ipecho.net/plain\")",
+      "got": "_requests_get.assert_called_with(settings.config[\"proxy_url\"]",
+      "got_raw": "_requests_get.assert_called_with(settings.config[\"proxy_url\"], proxies=expected_proxy)",
+      "exact_match": false,
+      "code_bleu": 0.10600313379512592,
+      "edit_similarity": 0.16091954022988506
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "200",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "404",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "data",
+      "got": "data",
+      "got_raw": "data",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "5",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "3",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "200",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "422",
+      "got": "422",
+      "got_raw": "422",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "404",
+      "got": "404",
+      "got_raw": "404",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "data",
+      "got": "data",
+      "got_raw": "data",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "8000",
+      "got": "8000",
+      "got_raw": "8000",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "schema",
+      "got": "schema",
+      "got_raw": "schema",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "2",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "6",
+      "got": "6",
+      "got_raw": "6",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "tags",
+      "got": "[{\"Key\":",
+      "got_raw": "[{\"Key\": \"cloudproxy\", \"Value\": \"true\"}]",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"ec2\")",
+      "got": "_boto3_resource.assert_called_with(\"ec2\"",
+      "got_raw": "_boto3_resource.assert_called_with(\"ec2\", region_name=\"us-east-1\")",
+      "exact_match": false,
+      "code_bleu": 0.2730120862709067,
+      "edit_similarity": 0.21739130434782608
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"spot\"",
+      "got": "\"spot\"",
+      "got_raw": "\"spot\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"stop\"",
+      "got": "\"terminate\"",
+      "got_raw": "\"terminate\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.35294117647058826
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"instance\"",
+      "got": "\"instance\"",
+      "got_raw": "\"instance\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"persistent\"",
+      "got": "\"persistent\"",
+      "got_raw": "\"persistent\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"mock-client\"",
+      "got": "\"mock-client\"",
+      "got_raw": "\"mock-client\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"mock-resource\"",
+      "got": "\"mock-resource\"",
+      "got_raw": "\"mock-resource\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "[mock_instance]",
+      "got": "\"i-12345\"",
+      "got_raw": "\"i-12345\"",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.08333333333333333
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "stopped_instances",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.09523809523809523
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "started_instances",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.19047619047619047
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "terminated_instances",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "result",
+      "got": "result",
+      "got_raw": "result",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "filedata",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "Exception)",
+      "got": "Exception)",
+      "got_raw": "Exception)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "str(exc_info.value)",
+      "got": "str(exc_info.value)",
+      "got_raw": "str(exc_info.value)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "DOFirewallExistsException)",
+      "got": "DOFirewallExistsException)",
+      "got_raw": "DOFirewallExistsException)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "True",
+      "got": "droplet",
+      "got_raw": "droplet",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "1234)",
+      "got": "1234)",
+      "got_raw": "1234)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"nyc1\"",
+      "got": "\"useast\"",
+      "got_raw": "\"useast\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "3164444",
+      "got": "1001",
+      "got_raw": "1001",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.18181818181818182
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "result_ids",
+      "got": "result_ids",
+      "got_raw": "result_ids",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "droplet_id)",
+      "got": "droplet_id)",
+      "got_raw": "droplet_id)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "'cloudproxy'",
+      "got": "'cloudproxy'",
+      "got_raw": "'cloudproxy'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "expected_ids",
+      "got": "expected_ids",
+      "got_raw": "expected_ids",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"s-1vcpu-1gb\"",
+      "got": "\"1gb\"",
+      "got_raw": "\"1gb\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5555555555555556
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "kwargs['tags']",
+      "got": "args[0].tags",
+      "got_raw": "args[0].tags",
+      "exact_match": false,
+      "code_bleu": 0.24446151121745047,
+      "edit_similarity": 0.6923076923076923
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "12345)",
+      "got": "12345)",
+      "got_raw": "12345)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "Exception, match=\"Some other error\")",
+      "got": "Exception)",
+      "got_raw": "Exception)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.43478260869565216
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "3164444",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "droplet_id)",
+      "got": "droplet_id)",
+      "got_raw": "droplet_id)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "[\"192.1.1.1\"]",
+      "got": "[\"192.1.1.1\"]",
+      "got_raw": "[\"192.1.1.1\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "2)",
+      "got": "2)",
+      "got_raw": "2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "[])",
+      "got": "True)",
+      "got_raw": "True)",
+      "exact_match": false,
+      "code_bleu": 0.35818997727451396,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "result",
+      "got": "result",
+      "got_raw": "result",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "body[\"name\"]",
+      "got": "body[\"name\"]",
+      "got_raw": "body[\"name\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"cloudproxy\"",
+      "got": "\"123\"",
+      "got_raw": "\"123\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.23529411764705882
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"ONE_TO_ONE_NAT\"",
+      "got": "\"INTERNAL_ONLY\"",
+      "got_raw": "\"INTERNAL_ONLY\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.45161290322580644
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"cloudproxy-123\"",
+      "got": "\"cloudproxy-123\"",
+      "got_raw": "\"cloudproxy-123\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "body[\"tags\"][\"items\"]",
+      "got": "body[\"tags\"]",
+      "got_raw": "body[\"tags\"]",
+      "exact_match": false,
+      "code_bleu": 0.5714285714285714,
+      "edit_similarity": 0.7272727272727273
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "{\"status\": \"RUNNING\"}",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.08
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "{\"status\": \"STOPPING\"}",
+      "got": "\"STOPPING\"",
+      "got_raw": "\"STOPPING\"",
+      "exact_match": false,
+      "code_bleu": 0.11892071150027211,
+      "edit_similarity": 0.625
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "3",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"hel1\"",
+      "got": "\"europe\"",
+      "got_raw": "\"europe\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.42857142857142855
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "result_ids",
+      "got": "result_ids",
+      "got_raw": "result_ids",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "mock_response",
+      "got": "\"cloudproxy-default-test-uuid\"",
+      "got_raw": "\"cloudproxy-default-test-uuid\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.18604651162790697
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"server-id-1\")",
+      "got": "\"server-id-1\")",
+      "got_raw": "\"server-id-1\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"delete-response\"",
+      "got": "\"delete-response\"",
+      "got_raw": "\"delete-response\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"type=cloudproxy\"",
+      "got": "{\"type\":",
+      "got_raw": "{\"type\": \"cloudproxy\"}",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.48
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "mock_client_instance",
+      "got": "mock_client_instance",
+      "got_raw": "mock_client_instance",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "2)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "[])",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"1.1.1.1\")",
+      "got": "mock_instance_config)",
+      "got_raw": "mock_instance_config)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.06451612903225806
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "[\"1.1.1.1\"])",
+      "got": "[\"1.1.1.1\"])",
+      "got_raw": "[\"1.1.1.1\"])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "mock_delete_queue)",
+      "got": "mock_delete_queue)",
+      "got_raw": "mock_delete_queue)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "mock_restart_queue)",
+      "got": "mock_restart_queue)",
+      "got_raw": "mock_restart_queue)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "mock_instance_config)",
+      "got": "mock_instance_config)",
+      "got_raw": "mock_instance_config)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "[])",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "delete_queue",
+      "got": "delete_queue",
+      "got_raw": "delete_queue",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "delete_queue)",
+      "got": "delete_queue)",
+      "got_raw": "delete_queue)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "2",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "functions",
+      "got": "functions",
+      "got_raw": "functions",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "Exception)",
+      "got": "Exception)",
+      "got_raw": "Exception)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "expected_ips",
+      "got": "expected_ips",
+      "got_raw": "expected_ips",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "test_config)",
+      "got": "test_config)",
+      "got_raw": "test_config)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "config",
+      "got": "config",
+      "got_raw": "config",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"t3.micro\"",
+      "got": "\"t3.micro\"",
+      "got_raw": "\"t3.micro\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"t2.small\"",
+      "got": "\"t2.small\"",
+      "got_raw": "\"t2.small\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"t2.large\"",
+      "got": "\"t2.large\"",
+      "got_raw": "\"t2.large\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"us-west1-b\"",
+      "got": "\"us-west1-b\"",
+      "got_raw": "\"us-west1-b\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"test-project\"",
+      "got": "\"test-project\"",
+      "got_raw": "\"test-project\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "config[\"providers\"]",
+      "got": "config[\"providers\"]",
+      "got_raw": "config[\"providers\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "config[\"providers\"][\"aws\"][\"instances\"][\"default\"][\"size\"]",
+      "got": "config[\"providers\"][\"aws\"][\"instances\"][\"default\"][\"size\"]",
+      "got_raw": "config[\"providers\"][\"aws\"][\"instances\"][\"default\"][\"size\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "config[\"providers\"][\"aws\"][\"instances\"][\"default\"][\"enabled\"]",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.03125,
+      "edit_similarity": 0.09230769230769231
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "387",
+      "got": "387",
+      "got_raw": "387",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"ewr\"",
+      "got": "\"ewr\"",
+      "got_raw": "\"ewr\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "False",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"tcp\"",
+      "got": "\"tcp\"",
+      "got_raw": "\"tcp\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"udp\"",
+      "got": "\"udp\"",
+      "got_raw": "\"udp\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"8899\"",
+      "got": "8899",
+      "got_raw": "8899",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"active\"",
+      "got": "\"active\"",
+      "got_raw": "\"active\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"test-id\"",
+      "got": "\"data['id']\"",
+      "got_raw": "\"data['id']\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.47619047619047616
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"testpass\")",
+      "got": "\"testpass\")",
+      "got_raw": "\"testpass\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"test-label\"",
+      "got": "\"test-label\"",
+      "got_raw": "\"test-label\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"vc2-1c-1gb\"",
+      "got": "\"vc2-1c-1gb\"",
+      "got_raw": "\"vc2-1c-1gb\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "1",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "3",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "result",
+      "got": "result",
+      "got_raw": "result",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"192.168.1.1\")",
+      "got": "\"192.168.1.1\")",
+      "got_raw": "\"192.168.1.1\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "8899",
+      "got": "8899",
+      "got_raw": "8899",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "now",
+      "got": "now",
+      "got_raw": "now",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "3600",
+      "got": "3600",
+      "got_raw": "3600",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "\"aws\"",
+      "got": "\"aws\"",
+      "got_raw": "\"aws\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "status",
+      "got": "status",
+      "got_raw": "status",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "state.pending",
+      "got": "state.pending",
+      "got_raw": "state.pending",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "claffin/cloudproxy",
+      "expected": "content",
+      "got": "content",
+      "got_raw": "content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "tx)",
+      "got": "tx)",
+      "got_raw": "tx)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "slug",
+      "got": "slug",
+      "got_raw": "slug",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "tags",
+      "got": "tags",
+      "got_raw": "tags",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "msg)",
+      "got": "msg)",
+      "got_raw": "msg)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "list)",
+      "got": "list)",
+      "got_raw": "list)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "dict)",
+      "got": "dict)",
+      "got_raw": "dict)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "func)",
+      "got": "func)",
+      "got_raw": "func)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "event",
+      "got": "event",
+      "got_raw": "event",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "result",
+      "got": "result",
+      "got_raw": "result",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "kwargs",
+      "got": "kwargs",
+      "got_raw": "kwargs",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "amount",
+      "got": "amount",
+      "got_raw": "amount",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "found)",
+      "got": "found)",
+      "got_raw": "found)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "agent)",
+      "got": "agent.id)",
+      "got_raw": "agent.id)",
+      "exact_match": false,
+      "code_bleu": 0.3976353643835253,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "events",
+      "got": "events",
+      "got_raw": "events",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "states",
+      "got": "states",
+      "got_raw": "states",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "config",
+      "got": "config",
+      "got_raw": "config",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "result",
+      "got": "\"body.user.email",
+      "got_raw": "\"body.user.email is not a valid email address\\nbody.items[0].price is not a valid decimal\"",
+      "exact_match": false,
+      "code_bleu": 0.18575057999133598,
+      "edit_similarity": 0.18181818181818182
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "chains)",
+      "got": "chains)",
+      "got_raw": "chains)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "prices)",
+      "got": "prices)",
+      "got_raw": "prices)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "post_id",
+      "got": "post_id",
+      "got_raw": "post_id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "profile",
+      "got": "profile",
+      "got_raw": "profile",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "user_id",
+      "got": "user_id",
+      "got_raw": "user_id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "headers",
+      "got": "headers",
+      "got_raw": "headers",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "returns",
+      "got": "returns",
+      "got_raw": "returns",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "token_a",
+      "got": "token_a",
+      "got_raw": "token_a",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "token_b",
+      "got": "token_b",
+      "got_raw": "token_b",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "address2",
+      "got": "address2",
+      "got_raw": "address2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "address3",
+      "got": "address3",
+      "got_raw": "address3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "address4",
+      "got": "address4",
+      "got_raw": "address4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "next_run",
+      "got": "next_run",
+      "got_raw": "next_run",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "agent_id",
+      "got": "user_id",
+      "got_raw": "user_id",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5333333333333333
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "account)",
+      "got": "account)",
+      "got_raw": "account)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "coin_data)",
+      "got": "coin_data)",
+      "got_raw": "coin_data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "fake_bytes",
+      "got": "fake_bytes",
+      "got_raw": "fake_bytes",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "mock_event",
+      "got": "mock_event",
+      "got_raw": "mock_event",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "[events[0]]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "[\"img1.jpg\"]",
+      "got": "[\"img1.jpg\"]",
+      "got_raw": "[\"img1.jpg\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "{\"chain\": chain})",
+      "got": "dummy.json())",
+      "got_raw": "dummy.json())",
+      "exact_match": false,
+      "code_bleu": 0.22089591134157885,
+      "edit_similarity": 0.13333333333333333
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "{\"fees\": \"data\"})",
+      "got": "{\"fees\": \"data\"})",
+      "got_raw": "{\"fees\": \"data\"})",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "{\"protocols\": []})",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": false,
+      "code_bleu": 0.27383275896268816,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "3",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "4",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "4",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "0",
+      "got": "1000",
+      "got_raw": "1000",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "0",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "1",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "0",
+      "got": "999",
+      "got_raw": "999",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"0\"",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"0\"",
+      "got": "\"0.00000\"",
+      "got_raw": "\"0.00000\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"add\"",
+      "got": "\"add\"",
+      "got_raw": "\"add\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"low\"",
+      "got": "\"medium\"",
+      "got_raw": "\"medium\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"ABC\"",
+      "got": "\"ABC\"",
+      "got_raw": "\"ABC\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"XYZ\"",
+      "got": "\"XYZ\"",
+      "got_raw": "\"XYZ\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"app\"",
+      "got": "\"app\"",
+      "got_raw": "\"app\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"safe\"",
+      "got": "\"privy\"",
+      "got_raw": "\"privy\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"KEEP\"",
+      "got": "\"KEEP\"",
+      "got_raw": "\"KEEP\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"POST\"",
+      "got": "\"POST\"",
+      "got_raw": "\"POST\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"USDC\"",
+      "got": "\"USDC\"",
+      "got_raw": "\"USDC\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"1.23\"",
+      "got": "\"1.230000\"",
+      "got_raw": "\"1.230000\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"0abc\"",
+      "got": "\"abc\"",
+      "got_raw": "\"abc\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"0def\"",
+      "got": "\"def\"",
+      "got_raw": "\"def\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"TRADE\"",
+      "got": "\"TRADE\"",
+      "got_raw": "\"TRADE\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"plain\"",
+      "got": "\"plain\"",
+      "got_raw": "\"plain\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"task1\"",
+      "got": "\"task1\"",
+      "got_raw": "\"task1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"task2\"",
+      "got": "\"task2\"",
+      "got_raw": "\"task2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"error\"",
+      "got": "\"ERROR\"",
+      "got_raw": "\"ERROR\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"erc20\"",
+      "got": "\"erc20\"",
+      "got_raw": "\"erc20\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"tests\"",
+      "got": "\"tests\"",
+      "got_raw": "\"tests\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"0xbbb\"",
+      "got": "\"0xbbb\"",
+      "got_raw": "\"0xbbb\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"0xabc\"",
+      "got": "\"0xabc\"",
+      "got_raw": "\"0xabc\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"0x123\"",
+      "got": "\"0x123\"",
+      "got_raw": "\"0x123\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"system\"",
+      "got": "\"system\"",
+      "got_raw": "\"system\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"post-1\"",
+      "got": "\"post-1\"",
+      "got_raw": "\"post-1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"123.45\"",
+      "got": "\"123.45\"",
+      "got_raw": "\"123.45\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"user_1\"",
+      "got": "\"user_1\"",
+      "got_raw": "\"user_1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"team_1\"",
+      "got": "\"team_1\"",
+      "got_raw": "\"team_1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"gpt-4o\"",
+      "got": "\"gpt-4o\"",
+      "got_raw": "\"gpt-4o\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"user_2\"",
+      "got": "\"user_2\"",
+      "got_raw": "\"user_2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"memory\"",
+      "got": "\"expense\"",
+      "got_raw": "\"expense\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.35294117647058826
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"task-1\"",
+      "got": "\"task-1\"",
+      "got_raw": "\"task-1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"post-1\"",
+      "got": "post.id",
+      "got_raw": "post.id",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.5333333333333333
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"failed\"",
+      "got": "\"failed\"",
+      "got_raw": "\"failed\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"0xabcd\"",
+      "got": "\"0x00abcd\"",
+      "got_raw": "\"0x00abcd\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"0xABCD\"",
+      "got": "\"0xABCD\"",
+      "got_raw": "\"0xABCD\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"signed\"",
+      "got": "\"signed\"",
+      "got_raw": "\"signed\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"unsafe\"",
+      "got": "\"unsafe\"",
+      "got_raw": "\"unsafe\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"agent-1\"",
+      "got": "\"agent-1\"",
+      "got_raw": "\"agent-1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"agent-1\"",
+      "got": "\"agent-1\"",
+      "got_raw": "\"agent-1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"DB Post\"",
+      "got": "\"DB Post\"",
+      "got_raw": "\"DB Post\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"gpt-3.5\"",
+      "got": "\"gpt-3.5\"",
+      "got_raw": "\"gpt-3.5\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"updated\"",
+      "got": "\"updated\"",
+      "got_raw": "\"updated\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"adjust\")",
+      "got": "mock_session)",
+      "got_raw": "mock_session)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.18181818181818182
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"event-3\"",
+      "got": "\"event-2\"",
+      "got_raw": "\"event-2\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"event-1\"",
+      "got": "\"event-1\"",
+      "got_raw": "\"event-1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"chat-1\")",
+      "got": "\"chat-1\")",
+      "got_raw": "\"chat-1\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"test123\"",
+      "got": "task.id",
+      "got_raw": "task.id",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"waiting\"",
+      "got": "\"waiting\"",
+      "got_raw": "\"waiting\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"agent-3\"",
+      "got": "\"agent-3\"",
+      "got_raw": "\"agent-3\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"kq_test\"",
+      "got": "\"kq_test\"",
+      "got_raw": "\"kq_test\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"0xprivy\"",
+      "got": "\"0xprivy\"",
+      "got_raw": "\"0xprivy\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"New Name\"",
+      "got": "\"New Name\"",
+      "got_raw": "\"New Name\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"DB Agent\"",
+      "got": "\"DB Agent\"",
+      "got_raw": "\"DB Agent\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"post-123\"",
+      "got": "\"post-123\"",
+      "got_raw": "\"post-123\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"DB Agent\"",
+      "got": "\"DB Agent\"",
+      "got_raw": "\"DB Agent\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"dev-user\"",
+      "got": "Decimal(\"0.5\")",
+      "got_raw": "Decimal(\"0.5\")",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"New Task\"",
+      "got": "\"New Task\"",
+      "got_raw": "\"New Task\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"NotFound\"",
+      "got": "\"post_not_found\"",
+      "got_raw": "\"post_not_found\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6153846153846154
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"1.234567\"",
+      "got": "\"1.234567\"",
+      "got_raw": "\"1.234567\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"wallet_1\"",
+      "got": "\"wallet_1\"",
+      "got_raw": "\"wallet_1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"wallet_2\"",
+      "got": "\"wallet_2\"",
+      "got_raw": "\"wallet_2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"0 * * * *\"",
+      "got": "\"0 * * * *\"",
+      "got_raw": "\"0 * * * *\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"video.mp4\"",
+      "got": "\"video.mp4\"",
+      "got_raw": "\"video.mp4\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"Test Post\"",
+      "got": "\"Test Post\"",
+      "got_raw": "\"Test Post\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"Test Task\"",
+      "got": "\"Test Task\"",
+      "got_raw": "\"Test Task\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"order-789\"",
+      "got": "\"order-789\"",
+      "got_raw": "\"order-789\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"wss://eth\"",
+      "got": "\"wss://eth\"",
+      "got_raw": "\"wss://eth\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"signed_tx\"",
+      "got": "\"signed_tx\"",
+      "got_raw": "\"signed_tx\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"Test Agent\"",
+      "got": "\"Test Agent\"",
+      "got_raw": "\"Test Agent\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"get-target\"",
+      "got": "\"get-target\"",
+      "got_raw": "\"get-target\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "\"Test Agent\"",
+      "got": "\"Test Agent\"",
+      "got_raw": "\"Test Agent\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "str(sql1)",
+      "got": "sql1",
+      "got_raw": "sql1",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.6153846153846154
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "str(sql2)",
+      "got": "str(sql2)",
+      "got_raw": "str(sql2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "str(sql3)",
+      "got": "str(sql3)",
+      "got_raw": "str(sql3)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "len(agents)",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "Decimal(\"0\")",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.15384615384615385
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "Decimal(\"0\")",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.15384615384615385
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "Decimal(\"2\")",
+      "got": "Decimal(\"2\")",
+      "got_raw": "Decimal(\"2\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "Decimal(\"2.5\")",
+      "got": "Decimal(\"2.5\")",
+      "got_raw": "Decimal(\"2.5\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "str(exc.value)",
+      "got": "str(exc.value)",
+      "got_raw": "str(exc.value)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "len(\"tx-\") + 8",
+      "got": "8",
+      "got_raw": "8",
+      "exact_match": false,
+      "code_bleu": 0.09910059291689341,
+      "edit_similarity": 0.13333333333333333
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "now.isoformat()",
+      "got": "now.isoformat()",
+      "got_raw": "now.isoformat()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "agent.id",
+      "got": "agent.id",
+      "got_raw": "agent.id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "agent.name",
+      "got": "agent.name",
+      "got_raw": "agent.name",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "agent.owner",
+      "got": "agent.owner",
+      "got_raw": "agent.owner",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "agent.model",
+      "got": "agent.model",
+      "got_raw": "agent.model",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "agent.prompt",
+      "got": "agent.prompt",
+      "got_raw": "agent.prompt",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "result.budget",
+      "got": "Decimal(\"0.0001\")",
+      "got_raw": "Decimal(\"0.0001\")",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "0.1, abs=1.0)",
+      "got": "0.1),",
+      "got_raw": "0.1),\n            app_id=None,\n            search_mode=False,\n            super_mode=False,\n        )",
+      "exact_match": false,
+      "code_bleu": 0.2686424829558855,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "OwnerType.USER",
+      "got": "CreditType.FEE",
+      "got_raw": "CreditType.FEE",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "Chain.Ethereum",
+      "got": "Chain.Ethereum",
+      "got_raw": "Chain.Ethereum",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "Chain.Arbitrum",
+      "got": "Chain.Arbitrum",
+      "got_raw": "Chain.Arbitrum",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "AuthorType.AGENT",
+      "got": "AuthorType.AGENT",
+      "got_raw": "AuthorType.AGENT",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "crestalnetwork/intentkit",
+      "expected": "OwnerType.PLATFORM",
+      "got": "OwnerType.USER",
+      "got_raw": "OwnerType.USER",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.6875
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "200",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "caplog.text",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "\"/path/to/obj/\"",
+      "got": "\"/path/to/obj/\"",
+      "got_raw": "\"/path/to/obj/\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "response.content",
+      "got": "response.content",
+      "got_raw": "response.content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "\"/accounts/profile/\"",
+      "got": "reverse(\"admin:tests_app_post_change\"",
+      "got_raw": "reverse(\"admin:tests_app_post_change\", args=[obj.id])",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.3157894736842105
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "\"/custom/success/path/\"",
+      "got": "\"/custom/success/path/\"",
+      "got_raw": "\"/custom/success/path/\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "\"path\"",
+      "got": "\"path\"",
+      "got_raw": "\"path\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "\"/path\"",
+      "got": "\"/path\"",
+      "got_raw": "\"/path\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "hash(\"path\")",
+      "got": "hash(asset)",
+      "got_raw": "hash(asset)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.6956521739130435
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "\"http://path\"",
+      "got": "\"http://path\"",
+      "got_raw": "\"http://path\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "\"/static/path\"",
+      "got": "\"Asset(path)\"",
+      "got_raw": "\"Asset(path)\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5925925925925926
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "\"https://path\"",
+      "got": "\"https://path\"",
+      "got_raw": "\"https://path\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "\"Asset: 'path'\"",
+      "got": "\"<Asset path>\"",
+      "got_raw": "\"<Asset path>\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8275862068965517
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "forms.Asset(\"path\")",
+      "got": "forms.Asset(\"path\")",
+      "got_raw": "forms.Asset(\"path\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "forms.Asset(\"other\")",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "'<script src=\"/static/path\" type=\"module\"></script>'",
+      "got": "\"ESM(path)\"",
+      "got_raw": "\"ESM(path)\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.19047619047619047
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "\"21\"",
+      "got": "\"10\"",
+      "got_raw": "\"10\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "\"bob\"",
+      "got": "\"eve\"",
+      "got_raw": "\"eve\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "\"eve\"",
+      "got": "\"eve\"",
+      "got_raw": "\"eve\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "response",
+      "got": "response",
+      "got_raw": "response",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "b\"<body></body>\"",
+      "got": "b\"<body></body>\"",
+      "got_raw": "b\"<body></body>\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "{\"request\": request})",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "b\"<body>HIJACKED</body>\"",
+      "got": "b\"HIJACKED\"",
+      "got_raw": "b\"HIJACKED\"",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6285714285714286
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "has_perm",
+      "got": "has_perm",
+      "got_raw": "has_perm",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "200",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "response.content",
+      "got": "response.content",
+      "got_raw": "response.content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "\"/\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "302",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "403",
+      "got": "403",
+      "got_raw": "403",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "400",
+      "got": "400",
+      "got_raw": "400",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "django-hijack/django-hijack",
+      "expected": "405",
+      "got": "405",
+      "got_raw": "405",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "3",
+      "got": "\"1\"",
+      "got_raw": "\"1\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "acct2",
+      "got": "acct2",
+      "got_raw": "acct2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "signed",
+      "got": "signed",
+      "got_raw": "signed",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "\"scrypt\"",
+      "got": "\"pbkdf2-hmac-sha256\"",
+      "got_raw": "\"pbkdf2-hmac-sha256\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.21428571428571427
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "\"pbkdf2\"",
+      "got": "\"pbkdf2\"",
+      "got_raw": "\"pbkdf2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "account2",
+      "got": "account2",
+      "got_raw": "account2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "TypeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "signed[\"v\"]",
+      "got": "v",
+      "got_raw": "v",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "signed[\"r\"]",
+      "got": "r",
+      "got_raw": "r",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "128",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "ZERO_BLOB",
+      "got": "ZERO_BLOB",
+      "got_raw": "ZERO_BLOB",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "blob_data_1",
+      "got": "blob_data_1",
+      "got_raw": "blob_data_1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "expected_blob",
+      "got": "expected_blob",
+      "got_raw": "expected_blob",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "expected_proofs",
+      "got": "expected_proofs",
+      "got_raw": "expected_proofs",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "HexBytes(go_tx_bytes)",
+      "got": "go_tx_bytes",
+      "got_raw": "go_tx_bytes",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.6875
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "model_dump",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "key",
+      "got": "key",
+      "got_raw": "key",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "a2.address",
+      "got": "a2.address",
+      "got_raw": "a2.address",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "expected_address",
+      "got": "expected_address",
+      "got_raw": "expected_address",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "ValidationError, match=\"Path is not valid.*\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.32142857142857145
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "ValidationError, match=\"Path.*is not valid.*\")",
+      "got": "ValidationError)",
+      "got_raw": "ValidationError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.5161290322580645
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "ValidationError, match=\"Language not detected .*\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.29508196721311475
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "ValidationError, match=\"Invalid choice for number of words.*\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.2465753424657534
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "ValidationError, match=\".* not a valid BIP39 mnemonic phrase!\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.24324324324324326
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "ValidationError, match=\"Invalid language choice 'pig_latin', must be one of.*\")",
+      "got": "pytest.raises(ValueError)",
+      "got_raw": " pytest.raises(ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.22089591134157885,
+      "edit_similarity": 0.17307692307692307
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "ValidationError)",
+      "got": "ValidationError)",
+      "got_raw": "ValidationError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "expected_signable",
+      "got": "expected_signable",
+      "got_raw": "expected_signable",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "64",
+      "got": "32",
+      "got_raw": "32",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "lang",
+      "got": "lang",
+      "got_raw": "lang",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "word",
+      "got": "word",
+      "got_raw": "word",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "expected_seed",
+      "got": "expected_seed",
+      "got_raw": "expected_seed",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "Language(lang)",
+      "got": "lang",
+      "got_raw": "lang",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "ValidationError)",
+      "got": "ValidationError)",
+      "got_raw": "ValidationError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "m.expand_word(\"\")",
+      "got": "m.expand_word(\"\")",
+      "got_raw": "m.expand_word(\"\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "expected_mnemonic",
+      "got": "expected_mnemonic",
+      "got_raw": "expected_mnemonic",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "m.expand_word(\" \")",
+      "got": "m.expand_word(\" \")",
+      "got_raw": "m.expand_word(\" \")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "m.expand_word(\"acb\")",
+      "got": "m.expand_word(\"acb\")",
+      "got_raw": "m.expand_word(\"acb\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "m.expand_word(\"acc\")",
+      "got": "m.expand_word(\"acc\")",
+      "got_raw": "m.expand_word(\"acc\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "m.expand_word(\"act\")",
+      "got": "m.expand_word(\"act\")",
+      "got_raw": "m.expand_word(\"act\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "Language.CHINESE_SIMPLIFIED",
+      "got": "Language.CHINESE_SIMPLIFIED",
+      "got_raw": "Language.CHINESE_SIMPLIFIED",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "Mnemonic.detect_language(word)",
+      "got": "Language.from_word(word)",
+      "got_raw": "Language.from_word(word)",
+      "exact_match": false,
+      "code_bleu": 0.4347208719449915,
+      "edit_similarity": 0.48148148148148145
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "unicode_decompose_string( expected_mnemonic )",
+      "got": "unicode_decompose_string(expected_mnemonic)",
+      "got_raw": "unicode_decompose_string(expected_mnemonic)",
+      "exact_match": false,
+      "code_bleu": 1.0,
+      "edit_similarity": 0.9772727272727273
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "new_local_account.sign_typed_data(full_message=message)",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "tuple(RLP_STRUCTURED_ACCESS_LIST)",
+      "got": "RLP_STRUCTURED_ACCESS_LIST",
+      "got_raw": "RLP_STRUCTURED_ACCESS_LIST",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.8813559322033898
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "tuple(RPC_STRUCTURED_ACCESS_LIST)",
+      "got": "RPC_STRUCTURED_ACCESS_LIST",
+      "got_raw": "RPC_STRUCTURED_ACCESS_LIST",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.8813559322033898
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "{\"testPydanticModel\": PYDANTIC_TEST_CLASS_JSON}",
+      "got": "PYDANTIC_TEST_CLASS_JSON",
+      "got_raw": "PYDANTIC_TEST_CLASS_JSON",
+      "exact_match": false,
+      "code_bleu": 0.11892071150027211,
+      "edit_similarity": 0.676056338028169
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "ValueError, match=\"provided object not formatted as rlp-structured access list\")",
+      "got": "pytest.raises(ValueError)",
+      "got_raw": " pytest.raises(ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.24446151121745047,
+      "edit_similarity": 0.20952380952380953
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "ValueError, match=\"provided object not formatted as JSON-RPC-structured access list\")",
+      "got": "pytest.raises(ValueError)",
+      "got_raw": " pytest.raises(ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.24446151121745047,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "expected.as_dict()",
+      "got": "expected.as_dict()",
+      "got_raw": "expected.as_dict()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "HexBytes(raw_transaction)",
+      "got": "HexBytes(raw_transaction)",
+      "got_raw": "HexBytes(raw_transaction)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "str(excinfo.value)",
+      "got": "excinfo.value.args[0]",
+      "got_raw": "excinfo.value.args[0]",
+      "exact_match": false,
+      "code_bleu": 0.2984745896009823,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "(TypeError, ABITypeError))",
+      "got": "ABITypeError)",
+      "got_raw": "ABITypeError)",
+      "exact_match": false,
+      "code_bleu": 0.23570226039551584,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "ValueError)",
+      "got": "ValidationError)",
+      "got_raw": "ValidationError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "ValidationError)",
+      "got": "ValidationError)",
+      "got_raw": "ValidationError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "len(set(string_test_cases))",
+      "got": "len(set(string_test_cases))",
+      "got_raw": "len(set(string_test_cases))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "encode_typed_data( *convert_to_3_arg(all_valid[message]) )",
+      "got": "all_valid[message]",
+      "got_raw": "all_valid[message]",
+      "exact_match": false,
+      "code_bleu": 0.36363636363636365,
+      "edit_similarity": 0.47368421052631576
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "hash_type_expected",
+      "got": "hash_type_expected",
+      "got_raw": "hash_type_expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "encode_type_expected",
+      "got": "encode_type_expected",
+      "got_raw": "encode_type_expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "encode_data_expected",
+      "got": "encode_data_expected",
+      "got_raw": "encode_data_expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "hash_struct_expected",
+      "got": "hash_struct_expected",
+      "got_raw": "hash_struct_expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "ethers_sig",
+      "got": "ethers_sig",
+      "got_raw": "ethers_sig",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "py_three_arg",
+      "got": "py_three_arg",
+      "got_raw": "py_three_arg",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "metamask_sig",
+      "got": "metamask_sig",
+      "got_raw": "metamask_sig",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ethereum/eth-account",
+      "expected": "ethers_address",
+      "got": "ethers_address",
+      "got_raw": "ethers_address",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "word",
+      "got": "word",
+      "got_raw": "word",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "wp_word",
+      "got": "wp_word",
+      "got_raw": "wp_word",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "X.shape",
+      "got": "X.shape",
+      "got_raw": "X.shape",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "len(nested_align)",
+      "got": "len(nested_align)",
+      "got_raw": "len(nested_align)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "align.data.shape[0]",
+      "got": "len(nested_align)",
+      "got_raw": "len(nested_align)",
+      "exact_match": false,
+      "code_bleu": 0.1510687698678384,
+      "edit_similarity": 0.2777777777777778
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "align.lengths.shape[0]",
+      "got": "len(spans)",
+      "got_raw": "len(spans)",
+      "exact_match": false,
+      "code_bleu": 0.1510687698678384,
+      "edit_similarity": 0.3125
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "tagger_trf",
+      "got": "tagger_trf",
+      "got_raw": "tagger_trf",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "\"custom_transformer\"",
+      "got": "\"custom_transformer\"",
+      "got_raw": "\"custom_transformer\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "wp_dict[key])",
+      "got": "wordpieces[key])",
+      "got_raw": "wordpieces[key])",
+      "exact_match": false,
+      "code_bleu": 0.7598356856515925,
+      "edit_similarity": 0.6896551724137931
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "aligned_wps",
+      "got": "aligned_wps",
+      "got_raw": "aligned_wps",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "1",
+      "got": "1000",
+      "got_raw": "1000",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "()",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "0.0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "[\"ner\"]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.39640237166757364,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "normal_tags",
+      "got": "normal_tags",
+      "got_raw": "normal_tags",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "512",
+      "got": "512",
+      "got_raw": "512",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "499",
+      "got": "499",
+      "got_raw": "499",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "[\"V\"]",
+      "got": "[\"V\"]",
+      "got_raw": "[\"V\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "[\"tagger\"]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.39640237166757364,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "6",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "\".\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "result",
+      "got": "result",
+      "got_raw": "result",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "\"One.\"",
+      "got": "\"One.\"",
+      "got_raw": "\"One.\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "\"One more.\"",
+      "got": "\"One more.\"",
+      "got_raw": "\"One more.\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "\"And one more.\"",
+      "got": "\"And one\"",
+      "got_raw": "\"And one\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "\"Three sentences in total.\"",
+      "got": "\"Three sentences in total.\"",
+      "got_raw": "\"Three sentences in total.\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "[\"textcat\"]",
+      "got": "[\"textcat\"]",
+      "got_raw": "[\"textcat\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "[\"tagger\"]",
+      "got": "[\"tagger\"]",
+      "got_raw": "[\"tagger\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "AssertionError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5384615384615384
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "max_length",
+      "got": "max_length",
+      "got_raw": "max_length",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "align.lengths.shape[0]",
+      "got": "len(sequences)",
+      "got_raw": "len(sequences)",
+      "exact_match": false,
+      "code_bleu": 0.1510687698678384,
+      "edit_similarity": 0.2777777777777778
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "wordpieces.strings[i][:max_length]",
+      "got": "truncated.strings[i]",
+      "got_raw": "truncated.strings[i]",
+      "exact_match": false,
+      "code_bleu": 0.4854640269421327,
+      "edit_similarity": 0.5185185185185185
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "ops.to_numpy(t2))",
+      "got": "yt.testing.assert_allclose(t1",
+      "got_raw": "yt.testing.assert_allclose(t1, t2)",
+      "exact_match": false,
+      "code_bleu": 0.20556680845025982,
+      "edit_similarity": 0.21739130434782608
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "1e-5",
+      "got": "1e-5",
+      "got_raw": "1e-5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "cats2",
+      "got": "cats2",
+      "got_raw": "cats2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "len(cats2)",
+      "got": "len(cats2)",
+      "got_raw": "len(cats2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "[\"transformer\", \"textcat\"]",
+      "got": "[\"transformer\", \"textcat\"]",
+      "got_raw": "[\"transformer\", \"textcat\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "explosion/spacy-transformers",
+      "expected": "[doc[0].tag_ for doc in docs2[:-1]]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.06995335970604241,
+      "edit_similarity": 0.10810810810810811
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0.1",
+      "got": "0.1",
+      "got_raw": "0.1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0.2",
+      "got": "0.2",
+      "got_raw": "0.2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "8",
+      "got": "8",
+      "got_raw": "8",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "16",
+      "got": "16",
+      "got_raw": "16",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "42",
+      "got": "42",
+      "got_raw": "42",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "75",
+      "got": "75",
+      "got_raw": "75",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "65",
+      "got": "65",
+      "got_raw": "65",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0.1",
+      "got": "0.1",
+      "got_raw": "0.1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0.2",
+      "got": "0.2",
+      "got_raw": "0.2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "i",
+      "got": "i",
+      "got_raw": "i",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0.2",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "65",
+      "got": "25",
+      "got_raw": "25",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "64",
+      "got": "1000",
+      "got_raw": "1000",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "31",
+      "got": "30",
+      "got_raw": "30",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "30",
+      "got": "30",
+      "got_raw": "30",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0.1",
+      "got": "0.1",
+      "got_raw": "0.1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0.2",
+      "got": "0.2",
+      "got_raw": "0.2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "4",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "52",
+      "got": "8",
+      "got_raw": "8",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "72",
+      "got": "100",
+      "got_raw": "100",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "82",
+      "got": "100",
+      "got_raw": "100",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "run_names",
+      "got": "run_names",
+      "got_raw": "run_names",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "summary[\"metrics\"]",
+      "got": "summary",
+      "got_raw": "summary",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.56
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "num_logs",
+      "got": "num_logs",
+      "got_raw": "num_logs",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "num_threads",
+      "got": "num_threads",
+      "got_raw": "num_threads",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "num_logs - 1",
+      "got": "num_logs - 1",
+      "got_raw": "num_logs - 1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "logs_per_thread",
+      "got": "logs_per_thread",
+      "got_raw": "logs_per_thread",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "num_threads * logs_per_thread",
+      "got": "num_threads * logs_per_thread",
+      "got_raw": "num_threads * logs_per_thread",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "max_abs",
+      "got": "max_abs",
+      "got_raw": "max_abs",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "channels",
+      "got": "channels",
+      "got_raw": "channels",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "SAMPLE_RATE",
+      "got": "SAMPLE_RATE",
+      "got_raw": "SAMPLE_RATE",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "RuntimeError, match=\"Call trackio.init\\\\(\\\\)\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.2807017543859649
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "ImportError, match=\"nvidia-ml-py is required\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.21052631578947367
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "65",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "64",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "31",
+      "got": "30",
+      "got_raw": "30",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "30",
+      "got": "30",
+      "got_raw": "30",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "11",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "10",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "len(data)",
+      "got": "12",
+      "got_raw": "12",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "hist_dict",
+      "got": "hist_dict",
+      "got_raw": "hist_dict",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "\"trackio.histogram\"",
+      "got": "\"Histogram\"",
+      "got_raw": "\"Histogram\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "ValueError, match=\"Must provide either sequence or np_histogram\")",
+      "got": "pytest.raises(ValueError)",
+      "got_raw": " pytest.raises(ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.24446151121745047,
+      "edit_similarity": 0.24444444444444444
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "ValueError, match=\"Cannot provide both sequence and np_histogram\")",
+      "got": "pytest.raises(ValueError, match=\"Must provide a sequence\")",
+      "got_raw": " pytest.raises(ValueError, match=\"Must provide a sequence\")",
+      "exact_match": false,
+      "code_bleu": 0.3672056269893592,
+      "edit_similarity": 0.6451612903225806
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "\"test_caption\"",
+      "got": "\"test_caption\"",
+      "got_raw": "\"test_caption\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "TrackioImage.TYPE",
+      "got": "\"image\"",
+      "got_raw": "\"image\"",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.4166666666666667
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "TrackioVideo.TYPE",
+      "got": "\"TrackioVideo\"",
+      "got_raw": "\"TrackioVideo\"",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.7741935483870968
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "TrackioAudio.TYPE",
+      "got": "\"TrackioAudio\"",
+      "got_raw": "\"TrackioAudio\"",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.7741935483870968
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "str(image._get_relative_file_path())",
+      "got": "str(Path(temp_dir)",
+      "got_raw": "str(Path(temp_dir) / \"test_run\" / \"0.jpg\")",
+      "exact_match": false,
+      "code_bleu": 0.24773732385103553,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "str(video._get_relative_file_path())",
+      "got": "str(temp_dir)",
+      "got_raw": "str(temp_dir)",
+      "exact_match": false,
+      "code_bleu": 0.25406637407730737,
+      "edit_similarity": 0.32653061224489793
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "str(audio._get_relative_file_path())",
+      "got": "str(Path(temp_dir)",
+      "got_raw": "str(Path(temp_dir) / \"test_run\" / \"0.wav\")",
+      "exact_match": false,
+      "code_bleu": 0.24773732385103553,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "3",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "4",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0.01",
+      "got": "0.01",
+      "got_raw": "0.01",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "\"proj\"",
+      "got": "\"proj\"",
+      "got_raw": "\"proj\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "\"run1\"",
+      "got": "run.id",
+      "got_raw": "run.id",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "metrics",
+      "got": "metrics",
+      "got_raw": "metrics",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "\"new-run\"",
+      "got": "\"new-run\"",
+      "got_raw": "\"new-run\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "\"testuser\"",
+      "got": "\"testuser\"",
+      "got_raw": "\"testuser\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "10",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "{}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0.9",
+      "got": "0.9",
+      "got_raw": "0.9",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "42",
+      "got": "42",
+      "got_raw": "42",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "84",
+      "got": "84",
+      "got_raw": "84",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "1.5",
+      "got": "1.5",
+      "got_raw": "1.5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "(0, 5)",
+      "got": "(0, 5)",
+      "got_raw": "(0, 5)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "(2, 0)",
+      "got": "(2, 10)",
+      "got_raw": "(2, 10)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.9230769230769231
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "len(df)",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "\"Unknown\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "test_path",
+      "got": "test_path",
+      "got_raw": "test_path",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "\"Just now\"",
+      "got": "\"30 seconds ago\"",
+      "got_raw": "\"30 seconds ago\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "\"1 hour ago\"",
+      "got": "\"1 hour ago\"",
+      "got_raw": "\"1 hour ago\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "\"2 days ago\"",
+      "got": "\"2 days ago\"",
+      "got_raw": "\"2 days ago\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "len(set(names))",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.07142857142857142,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "gradio-app/trackio",
+      "expected": "\"/data/trackio\"",
+      "got": "tmpdir",
+      "got_raw": "tmpdir",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.19047619047619047
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "5",
+      "got": "\"EMPIRE\"",
+      "got_raw": "\"EMPIRE\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "0",
+      "got": "\"test\"",
+      "got_raw": "\"test\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "2",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "e",
+      "got": "e",
+      "got_raw": "e",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "k",
+      "got": "k",
+      "got_raw": "k",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "r",
+      "got": "r",
+      "got_raw": "r",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "s",
+      "got": "r",
+      "got_raw": "r",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "m",
+      "got": "obj1",
+      "got_raw": "obj1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "v",
+      "got": "v",
+      "got_raw": "v",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "s",
+      "got": "obj1",
+      "got_raw": "obj1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "e2",
+      "got": "e2",
+      "got_raw": "e2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "e1",
+      "got": "e1",
+      "got_raw": "e1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "sdl",
+      "got": "\"",
+      "got_raw": "\" \"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "out",
+      "got": "out",
+      "got_raw": "out",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "loc",
+      "got": "loc",
+      "got_raw": "loc",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "res",
+      "got": "res",
+      "got_raw": "res",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "ast",
+      "got": "ast",
+      "got_raw": "ast",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "ast",
+      "got": "ast_before_print_call",
+      "got_raw": "ast_before_print_call",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "key",
+      "got": "key",
+      "got_raw": "key",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "ast",
+      "got": "ast",
+      "got_raw": "ast",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "str",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "obj",
+      "got": "obj",
+      "got_raw": "obj",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "res",
+      "got": "{\"kind\":",
+      "got_raw": "{\"kind\": \"name\", \"value\": \"test\", \"locations\": []}",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "ast",
+      "got": "ast",
+      "got_raw": "ast",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "sdl",
+      "got": "sdl",
+      "got_raw": "sdl",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "sdl",
+      "got": "sdl",
+      "got_raw": "sdl",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "type",
+      "got": "type(Exception)",
+      "got_raw": "type(Exception)",
+      "exact_match": false,
+      "code_bleu": 0.35930411196308426,
+      "edit_similarity": 0.42105263157894735
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "args",
+      "got": "tuple(args.values())[:2]",
+      "got_raw": "tuple(args.values())[:2]",
+      "exact_match": false,
+      "code_bleu": 0.10390302174233558,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "()",
+      "got": "()",
+      "got_raw": "()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "{}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "()",
+      "got": "()",
+      "got_raw": "()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[]",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "()",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "()",
+      "got": "schema.directives",
+      "got_raw": "schema.directives",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[]",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[4]",
+      "got": "[0]",
+      "got_raw": "[0]",
+      "exact_match": false,
+      "code_bleu": 0.49999999999999994,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[0]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.39640237166757364,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[6]",
+      "got": "[6]",
+      "got_raw": "[6]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[2]",
+      "got": "[3]",
+      "got_raw": "[3]",
+      "exact_match": false,
+      "code_bleu": 0.49999999999999994,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[0]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.39640237166757364,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "{})",
+      "got": "123)",
+      "got_raw": "123)",
+      "exact_match": false,
+      "code_bleu": 0.35818997727451396,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[])",
+      "got": "[[]]",
+      "got_raw": "[[]]",
+      "exact_match": false,
+      "code_bleu": 0.47287080450158786,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[42]",
+      "got": "[42]",
+      "got_raw": "[42]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[\"\"]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.39640237166757364,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "(2, 3)",
+      "got": "SourceLocation(2, 3)",
+      "got_raw": "SourceLocation(2, 3)",
+      "exact_match": false,
+      "code_bleu": 0.8091067115702212,
+      "edit_similarity": 0.46153846153846156
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "(1, 1)",
+      "got": "SourceLocation(0, 0)",
+      "got_raw": "SourceLocation(0, 0)",
+      "exact_match": false,
+      "code_bleu": 0.2626909894424158,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "(2, 6)",
+      "got": "SourceLocation(1, 1)",
+      "got_raw": "SourceLocation(1, 1)",
+      "exact_match": false,
+      "code_bleu": 0.2626909894424158,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[[42]]",
+      "got": "[42]",
+      "got_raw": "[42]",
+      "exact_match": false,
+      "code_bleu": 0.5045378491522287,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[True]",
+      "got": "[True]",
+      "got_raw": "[True]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "(1, 2))",
+      "got": "(0, 0))",
+      "got_raw": "(0, 0))",
+      "exact_match": false,
+      "code_bleu": 0.3303164318013807,
+      "edit_similarity": 0.7142857142857143
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "(1, 4))",
+      "got": "(1, 3))",
+      "got_raw": "(1, 3))",
+      "exact_match": false,
+      "code_bleu": 0.488923022434901,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "(1, 5))",
+      "got": "(1, 5))",
+      "got_raw": "(1, 5))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "(1, 1))",
+      "got": "\"Unterminated string.\"",
+      "got_raw": "\"Unterminated string.\"",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.06896551724137931
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "(1, 7))",
+      "got": "(1, 1))",
+      "got_raw": "(1, 1))",
+      "exact_match": false,
+      "code_bleu": 0.488923022434901,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[(1, 3)]",
+      "got": "[(1, 3)]",
+      "got_raw": "[(1, 3)]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[2, 4, 6]",
+      "got": "[2, 4, 6]",
+      "got_raw": "[2, 4, 6]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[0, 1, 2]",
+      "got": "[0, 1, 2, 3]",
+      "got_raw": "[0, 1, 2, 3]",
+      "exact_match": false,
+      "code_bleu": 0.6606328636027614,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[2, 1, 0]",
+      "got": "[0, 1, 2, 3]",
+      "got_raw": "[0, 1, 2, 3]",
+      "exact_match": false,
+      "code_bleu": 0.3356891925037239,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[1, 2, 3]",
+      "got": "[1, 2, 3]",
+      "got_raw": "[1, 2, 3]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[\"\", \"a\"]",
+      "got": "[\"a\"]",
+      "got_raw": "[\"a\"]",
+      "exact_match": false,
+      "code_bleu": 0.38336586254776345,
+      "edit_similarity": 0.7142857142857143
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "({}, None)",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.23570226039551584,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "({}, None)",
+      "got": "{ \"errors\": [ { \"message\": \"Field thisIsIllegalDoNotIncludeMe is not supported\", \"locations\": [ { \"line\": 1, \"column\": 17, } ], \"extensions\": { \"code\": \"ILLEGAL_FIELD\", }, }, ], } ] }",
+      "got_raw": "{ \"errors\": [ { \"message\": \"Field thisIsIllegalDoNotIncludeMe is not supported\", \"locations\": [ { \"line\": 1, \"column\": 17, } ], \"extensions\": { \"code\": \"ILLEGAL_FIELD\", }, }, ], } ] }",
+      "exact_match": false,
+      "code_bleu": 0.04259410369301382,
+      "edit_similarity": 0.04145077720207254
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[0, \"one\"]",
+      "got": "[0, \"one\"]",
+      "got_raw": "[0, \"one\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "{\"foo\": 5}",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": false,
+      "code_bleu": 0.11892071150027211,
+      "edit_similarity": 0.18181818181818182
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "{\"foo\": 7}",
+      "got": "7",
+      "got_raw": "7",
+      "exact_match": false,
+      "code_bleu": 0.11892071150027211,
+      "edit_similarity": 0.18181818181818182
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "(data, None)",
+      "got": "(data, errors, True)",
+      "got_raw": "(data, errors, True)",
+      "exact_match": false,
+      "code_bleu": 0.3655552228545123,
+      "edit_similarity": 0.625
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[foo_object]",
+      "got": "[foo_object]",
+      "got_raw": "[foo_object]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "(data, None)",
+      "got": "(data, errors)",
+      "got_raw": "(data, errors)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6923076923076923
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "(data, None)",
+      "got": "ExecutionResult(data={\"null\": None})",
+      "got_raw": "ExecutionResult(data={\"null\": None})",
+      "exact_match": false,
+      "code_bleu": 0.18850319022747347,
+      "edit_similarity": 0.4583333333333333
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "{\"foo\": 123}",
+      "got": "123",
+      "got_raw": "123",
+      "exact_match": false,
+      "code_bleu": 0.11892071150027211,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "{\"id\": \"foo\"}",
+      "got": "\"foo\"",
+      "got_raw": "\"foo\"",
+      "exact_match": false,
+      "code_bleu": 0.11892071150027211,
+      "edit_similarity": 0.5555555555555556
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[\"listField\"]",
+      "got": "[\"apple\"",
+      "got_raw": "[\"apple\", \"banana\", \"coconut\"]",
+      "exact_match": false,
+      "code_bleu": 0.35818997727451396,
+      "edit_similarity": 0.47619047619047616
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[0, \"one\", 2]",
+      "got": "[0, \"one\", 2]",
+      "got_raw": "[0, \"one\", 2]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "{\"foo\": None}",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": false,
+      "code_bleu": 0.11892071150027211,
+      "edit_similarity": 0.47058823529411764
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "{\"data\": data}",
+      "got": "formatted",
+      "got_raw": "formatted",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.2608695652173913
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "(data, errors)",
+      "got": "(data, errors)",
+      "got_raw": "(data, errors)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "(None, errors)",
+      "got": "(data, errors, None)",
+      "got_raw": "(data, errors, None)",
+      "exact_match": false,
+      "code_bleu": 0.29071536848410967,
+      "edit_similarity": 0.5882352941176471
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "[data, errors]",
+      "got": "[1, 2, 3]",
+      "got_raw": "[1, 2, 3]",
+      "exact_match": false,
+      "code_bleu": 0.22089591134157885,
+      "edit_similarity": 0.34782608695652173
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "(None, errors)",
+      "got": "({\"listField\": [1, 2]}, None)",
+      "got_raw": "({\"listField\": [1, 2]}, None)",
+      "exact_match": false,
+      "code_bleu": 0.12011055432195765,
+      "edit_similarity": 0.27906976744186046
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "{\"bar\": \"rab\"}",
+      "got": "\"oofbar\"",
+      "got_raw": "\"oofbar\"",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.45454545454545453
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "{\"bar\": \"foo\"}",
+      "got": "{\"oof\": \"foof\"}",
+      "got_raw": "{\"oof\": \"foof\"}",
+      "exact_match": false,
+      "code_bleu": 0.32466791547509893,
+      "edit_similarity": 0.7586206896551724
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "{\"hasNext\": True}",
+      "got": "{\"hasNext\": True}",
+      "got_raw": "{\"hasNext\": True}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "({\"a\": {}}, None)",
+      "got": "{\"a\": {\"b\": \"c\"}}",
+      "got_raw": "{\"a\": {\"b\": \"c\"}}",
+      "exact_match": false,
+      "code_bleu": 0.40662090162443015,
+      "edit_similarity": 0.5294117647058824
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "{\"test\": \"test\"})",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.10526315789473684
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "{\"test\": \"test\"})",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": false,
+      "code_bleu": 0.2259005009024612,
+      "edit_similarity": 0.1
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "{\"hasNext\": False}",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "({\"a\": \"a\"}, None)",
+      "got": "ExecutionResult(data={\"a\": \"1\"})",
+      "got_raw": "ExecutionResult(data={\"a\": \"1\"})",
+      "exact_match": false,
+      "code_bleu": 0.26985534666825095,
+      "edit_similarity": 0.44
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "({\"a\": \"b\"}, None)",
+      "got": "{\"a\": \"b\"}",
+      "got_raw": "{\"a\": \"b\"}",
+      "exact_match": false,
+      "code_bleu": 0.5555555555555556,
+      "edit_similarity": 0.7142857142857143
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "({\"c\": \"d\"}, None)",
+      "got": "{\"c\": \"d\"}",
+      "got_raw": "{\"c\": \"d\"}",
+      "exact_match": false,
+      "code_bleu": 0.5555555555555556,
+      "edit_similarity": 0.7142857142857143
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "key(\"a\")",
+      "got": "key(\"a\")",
+      "got_raw": "key(\"a\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "key(\"1\")",
+      "got": "key(\"b\")",
+      "got_raw": "key(\"b\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.875
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "key(\"2\")",
+      "got": "key(\"2\")",
+      "got_raw": "key(\"2\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "key(\"AA\")",
+      "got": "key(\"B\")",
+      "got_raw": "key(\"B\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.8235294117647058
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "key(\"11\")",
+      "got": "key(\"10\")",
+      "got_raw": "key(\"10\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "key(\"00\")",
+      "got": "key(\"1\")",
+      "got_raw": "key(\"1\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.8235294117647058
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "key(\"A1A\")",
+      "got": "key(\"A\")",
+      "got_raw": "key(\"A\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "key(\"200\")",
+      "got": "key(\"012\")",
+      "got_raw": "key(\"012\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "key(\"a9a\")",
+      "got": "key(\"a1a\")",
+      "got_raw": "key(\"a1a\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.9
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "hash(None)",
+      "got": "hash(UndefinedType)",
+      "got_raw": "hash(UndefinedType)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.5517241379310345
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "hash(True)",
+      "got": "hash(True)",
+      "got_raw": "hash(True)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "key(\"a09a\")",
+      "got": "key(\"a0a\")",
+      "got_raw": "key(\"a0a\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "key(\"a0a9\")",
+      "got": "key(\"a0a2\")",
+      "got_raw": "key(\"a0a2\")",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "hash(False)",
+      "got": "hash(1)",
+      "got_raw": "hash(1)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "test_value())",
+      "got": "_values(test_value()",
+      "got_raw": "_values(test_value(), test_value())",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.7272727272727273
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"\"",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"n\"",
+      "got": "\"n\"",
+      "got_raw": "\"n\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"A\"",
+      "got": "[\"A\"]",
+      "got_raw": "[\"A\"]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"0\"",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"\")",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"l2\"",
+      "got": "\"l1\"",
+      "got_raw": "\"l1\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"[]\"",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"()\"",
+      "got": "()",
+      "got_raw": "()",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"{}\"",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"foo\"",
+      "got": "\"foo\"",
+      "got_raw": "\"foo\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"baz\"",
+      "got": "\"custom_value\"",
+      "got_raw": "\"custom_value\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3157894736842105
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "'\" \"'",
+      "got": "\" \"",
+      "got_raw": "\" \"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"foo\"",
+      "got": "\"foo\"",
+      "got_raw": "\"foo\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"foo\"",
+      "got": "\"foo\"",
+      "got_raw": "\"foo\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"bar\"",
+      "got": "\"bar\"",
+      "got_raw": "\"bar\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"Foo\"",
+      "got": "\"Foo\"",
+      "got_raw": "\"Foo\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"0.0\"",
+      "got": "\"0.0\"",
+      "got_raw": "\"0.0\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"314\"",
+      "got": "\"314\"",
+      "got_raw": "\"314\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"nan\"",
+      "got": "\"nan\"",
+      "got_raw": "\"nan\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"inf\"",
+      "got": "\"inf\"",
+      "got_raw": "\"inf\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"Int\"",
+      "got": "{",
+      "got_raw": "{ \"name\": \"Int\", \"description\": None, \"fields\": None, \"interfaces\": [], \"is_interface\": False, \"is_object\": True, \"is_scalar\": False, \"is_enum\": False, \"is_input_object\": False, \"is_interface_of\": [], \"is_interface_implemented_by\": [], \"is_interface_implemented_by_directives\": [], \"is_interface_implemented_by_fields\": [], \"is_interface_implemented_by_fields_directives\": [], \"is_interface_implemented_by_fields_args\": [], \"is_interface_implemented_by_fields_args_directives\": [], \"is_interface_implemented_by_fields_args_values\":",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"two\"",
+      "got": "\"two\"",
+      "got_raw": "\"two\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"[1]\"",
+      "got": "\"1\"",
+      "got_raw": "\"1\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"Foo\"",
+      "got": "\"Foo\"",
+      "got_raw": "\"Foo\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"foo\"",
+      "got": "\"foo\"",
+      "got_raw": "\"foo\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"loc\"",
+      "got": "\"loc\"",
+      "got_raw": "\"loc\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"1.5\"",
+      "got": "\"1.5\"",
+      "got_raw": "\"1.5\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"Cat\"",
+      "got": "\"Cat\"",
+      "got_raw": "\"Cat\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"lazy\"",
+      "got": "\"lazy\"",
+      "got_raw": "\"lazy\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"ouch\"",
+      "got": "\"ouch\"",
+      "got_raw": "\"ouch\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"None\"",
+      "got": "\"None\"",
+      "got_raw": "\"None\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"True\"",
+      "got": "\"True\"",
+      "got_raw": "\"True\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\".one\"",
+      "got": "\"one\"",
+      "got_raw": "\"one\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"test\"",
+      "got": "\"test\"",
+      "got_raw": "\"test\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"@foo\"",
+      "got": "\"foo\"",
+      "got_raw": "\"foo\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"kind\"",
+      "got": "\"kind\"",
+      "got_raw": "\"kind\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"alpha\"",
+      "got": "\"alpha\"",
+      "got_raw": "\"alpha\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"1.2.3\"",
+      "got": "\"1.2.3\"",
+      "got_raw": "\"1.2.3\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"Query\"",
+      "got": "\"Lazy\"",
+      "got_raw": "\"Lazy\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.46153846153846156
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"First\"",
+      "got": "\"First\"",
+      "got_raw": "\"First\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"query\"",
+      "got": "{\"value\":",
+      "got_raw": "{\"value\": \"Query\"}",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "'\"foo\"'",
+      "got": "'\"foo\"'",
+      "got_raw": "'\"foo\"'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"A or B\"",
+      "got": "\"A or B\"",
+      "got_raw": "\"A or B\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"Second\"",
+      "got": "\"Second\"",
+      "got_raw": "\"Second\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"1.2.3a4\"",
+      "got": "\"1.2.3a4\"",
+      "got_raw": "\"1.2.3a4\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"Goodbye\"",
+      "got": "\"Goodbye\"",
+      "got_raw": "\"Goodbye\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"A and B\"",
+      "got": "\"A AND B\"",
+      "got_raw": "\"A AND B\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"1.2.3rc4\"",
+      "got": "\"1.2.3rc4\"",
+      "got_raw": "\"1.2.3rc4\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"resolved\"",
+      "got": "\"resolved\"",
+      "got_raw": "\"resolved\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "'\"\\u21bb\"'",
+      "got": "\"\\u21bb\"",
+      "got_raw": "\"\\u21bb\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"not lazy\"",
+      "got": "\"not lazy\"",
+      "got_raw": "\"not lazy\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "'\"\\\\u0000\"'",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"SnakeCase\"",
+      "got": "\"SnakeCase\"",
+      "got_raw": "\"SnakeCase\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"CamelCase\"",
+      "got": "\"CamelCase\"",
+      "got_raw": "\"CamelCase\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"snakeCase\"",
+      "got": "\"SnakeCase\"",
+      "got_raw": "\"SnakeCase\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"lazyField\"",
+      "got": "\"lazyField\"",
+      "got_raw": "\"lazyField\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"Undefined\"",
+      "got": "\"Undefined\"",
+      "got_raw": "\"Undefined\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\">--()-->\")",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.15384615384615385
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"type Bar\")",
+      "got": "schema)",
+      "got_raw": "schema)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"type Foo\")",
+      "got": "schema)",
+      "got_raw": "schema)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "\"Some error\"",
+      "got": "\"Some error\"",
+      "got_raw": "\"Some error\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "True",
+      "got": "{\"foo\":",
+      "got_raw": "{\"foo\": \"FooValue\"}",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "1 + 2j",
+      "got": "1 + 2j",
+      "got_raw": "1 + 2j",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "*items)",
+      "got": "*items)",
+      "got_raw": "*items)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "graphql-python/graphql-core",
+      "expected": "values[:-1]",
+      "got": "values",
+      "got_raw": "values",
+      "exact_match": false,
+      "code_bleu": 0.09910059291689341,
+      "edit_similarity": 0.7058823529411765
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "5",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "2.5",
+      "got": "2.5",
+      "got_raw": "2.5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "result",
+      "got": "result",
+      "got_raw": "result",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "\"echo_tool\"",
+      "got": "\"Echo",
+      "got_raw": "\"Echo Server\"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "\"echo_tool\"",
+      "got": "\"echo_tool\"",
+      "got_raw": "\"echo_tool\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "\"Echo: hello\"",
+      "got": "\"Echo the input text\"",
+      "got_raw": "\"Echo the input text\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5294117647058824
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "\"Echo: empty\"",
+      "got": "\"empty\"",
+      "got_raw": "\"empty\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "\"def_\"",
+      "got": "\"def\"",
+      "got_raw": "\"def\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "result_int",
+      "got": "result_int",
+      "got_raw": "result_int",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "\"echo_tool\"",
+      "got": "\"echo_tool\"",
+      "got_raw": "\"echo_tool\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "\"Echo: hello\"",
+      "got": "\"Echo: hello\"",
+      "got_raw": "\"Echo: hello\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "result_string",
+      "got": "result_string",
+      "got_raw": "result_string",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "70",
+      "got": "70",
+      "got_raw": "70",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "\"def_\"",
+      "got": "\"def\"",
+      "got_raw": "\"def\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "\"sunny\"",
+      "got": "\"sunny\"",
+      "got_raw": "\"sunny\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "(256, 256)",
+      "got": "(100, 100)",
+      "got_raw": "(100, 100)",
+      "exact_match": false,
+      "code_bleu": 0.32466791547509893,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "\"echo_tool\"",
+      "got": "\"echo_tool\"",
+      "got_raw": "\"echo_tool\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "\"test_image\"",
+      "got": "\"test_image\"",
+      "got_raw": "\"test_image\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "\"test_audio\"",
+      "got": "\"test_audio\"",
+      "got_raw": "\"test_audio\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "\"Echo: hello\"",
+      "got": "\"Echo: hello\"",
+      "got_raw": "\"Echo: hello\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "42",
+      "got": "42",
+      "got_raw": "42",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "grll/mcpadapt",
+      "expected": "\"test\"",
+      "got": "\"test\"",
+      "got_raw": "\"test\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "serialized",
+      "got": "serialized",
+      "got_raw": "serialized",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "expected_attrs",
+      "got": "expected_attrs",
+      "got_raw": "expected_attrs",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "'<svg><pattern id=\"patt1\" href=\"",
+      "got": "\"patt1 xlink:href=\\\"",
+      "got_raw": "\"patt1 xlink:href=\\\"#patt2\\\" id=\\\"patt1\\\" />\"",
+      "exact_match": false,
+      "code_bleu": 0.1511944296260267,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "OrderedDict([ ((None, 'id'), 'patt1'), (('http://www.w3.org/1999/xlink', 'href'), '",
+      "got": "{ 'id': 'patt1', 'href': '",
+      "got_raw": "{ 'id': 'patt1', 'href': '#patt2' }",
+      "exact_match": false,
+      "code_bleu": 0.09561379521515795,
+      "edit_similarity": 0.42201834862385323
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "1024",
+      "got": "1024",
+      "got_raw": "1024",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "10240",
+      "got": "10240",
+      "got_raw": "10240",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "\"big5\"",
+      "got": "\"Big5\"",
+      "got_raw": "\"Big5\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8333333333333334
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "TypeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "\"Caf\\u00E9\"",
+      "got": "\"Caf\\u00E9\"",
+      "got_raw": "\"Caf\\u00E9\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "p.documentEncoding",
+      "got": "doc.encoding",
+      "got_raw": "doc.encoding",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.7333333333333333
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "stream.charEncoding[0].name",
+      "got": "stream.encoding",
+      "got_raw": "stream.encoding",
+      "exact_match": false,
+      "code_bleu": 0.2229763340630102,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "r",
+      "got": "r",
+      "got_raw": "r",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "2",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "\"html\"",
+      "got": "\"html\"",
+      "got_raw": "\"html\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "namespaces[\"html\"]",
+      "got": "\"html\"",
+      "got_raw": "\"html\"",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "\"{%s}html\" % (namespaces[\"html\"],)",
+      "got": "\"html\"",
+      "got_raw": "\"html\"",
+      "exact_match": false,
+      "code_bleu": 0.0660670619445956,
+      "edit_similarity": 0.3
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "sanitized",
+      "got": "sanitized",
+      "got_raw": "sanitized",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "sanitize_html(input)",
+      "got": "input",
+      "got_raw": "input",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "attrs",
+      "got": "attrs",
+      "got_raw": "attrs",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "result",
+      "got": "result",
+      "got_raw": "result",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "TypeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "UnicodeEncodeError)",
+      "got": "UnicodeError)",
+      "got_raw": "UnicodeError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.8125
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "'<!DOCTYPE html SYSTEM \"about:legacy-compat\"><html>&gt;</html>'",
+      "got": "'<!DOCTYPE html SYSTEM \"about:legacy-compat\"><html>&gt;</html>'",
+      "got_raw": "'<!DOCTYPE html SYSTEM \"about:legacy-compat\"><html>&gt;</html>'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "'<!DOCTYPE html SYSTEM \"about:legacy-compat\"><html>\\u03B2</html>'",
+      "got": "'<html>&beta;</html>'",
+      "got_raw": "'<html>&beta;</html>'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3488372093023256
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "'<!DOCTYPE html SYSTEM \"about:legacy-compat\"><html>&beta;</html>'",
+      "got": "'<!DOCTYPE html SYSTEM \"about:legacy-compat\"><html>\\u03B2</html>'",
+      "got_raw": "'<!DOCTYPE html SYSTEM \"about:legacy-compat\"><html>\\u03B2</html>'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9076923076923077
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "s",
+      "got": "s",
+      "got_raw": "s",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "6",
+      "got": "6",
+      "got_raw": "6",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "b\"\"",
+      "got": "b\"\"",
+      "got_raw": "b\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "\"'\"",
+      "got": "'?'",
+      "got_raw": "'?'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "\"x\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "\"a\"",
+      "got": "\"n\"",
+      "got_raw": "\"n\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "\"b\"",
+      "got": "\"b\"",
+      "got_raw": "\"b\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "\"c\"",
+      "got": "\"c\"",
+      "got_raw": "\"c\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "\"d\"",
+      "got": "\"d\"",
+      "got_raw": "\"d\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "num",
+      "got": "num",
+      "got_raw": "num",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "1",
+      "got": "len(attrs)",
+      "got_raw": "len(attrs)",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "out_name",
+      "got": "out_name",
+      "got_raw": "out_name",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "out_value",
+      "got": "out_value",
+      "got_raw": "out_value",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "len(attrs)",
+      "got": "len(attrs)",
+      "got_raw": "len(attrs)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "[('a', '1')]",
+      "got": "[('a', '1')]",
+      "got_raw": "[('a', '1')]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "tokenTypes['StartTag']",
+      "got": "tokenTypes['StartTag']",
+      "got_raw": "tokenTypes['StartTag']",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "handler.visited",
+      "got": "handler.traces",
+      "got_raw": "handler.traces",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.6896551724137931
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "\"EndTag\"",
+      "got": "'EndTag'",
+      "got_raw": "'EndTag'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "out_name",
+      "got": "(out_name",
+      "got_raw": "(out_name, out_value)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.9411764705882353
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "out_value",
+      "got": "out_value",
+      "got_raw": "out_value",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "'StartTag'",
+      "got": "\"text\"",
+      "got_raw": "\"text\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "outputToken",
+      "got": "outputToken",
+      "got_raw": "outputToken",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "[((None, 'a'), '1'), ((None, 'refX'), '2'), ((None, 'b'), '3'), (('http://www.w3.org/XML/1998/namespace', 'lang'), '4'), ((None, 'c'), '5')]",
+      "got": "[('a', '1'), ('refx', '2'), ('b', '3'), ('xml:lang', '4'), ('c', '5')]",
+      "got_raw": "[('a', '1'), ('refx', '2'), ('b', '3'), ('xml:lang', '4'), ('c', '5')]",
+      "exact_match": false,
+      "code_bleu": 0.34965660677964516,
+      "edit_similarity": 0.6285714285714286
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "output",
+      "got": "output",
+      "got_raw": "output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "output",
+      "got": "output",
+      "got_raw": "output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "html5lib/html5lib-python",
+      "expected": "len(expectedErrors)",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "30",
+      "got": "30",
+      "got_raw": "30",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "70",
+      "got": "70",
+      "got_raw": "70",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "100",
+      "got": "100",
+      "got_raw": "100",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "['test']",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.39640237166757364,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "SystemExit)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.2727272727272727
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "[minimal_event]",
+      "got": "[",
+      "got_raw": "[ (minimal_event['id'], minimal_event_overlapping['id']) ]",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.125
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "0",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "(5, 5)",
+      "got": "test_day_width",
+      "got_raw": "test_day_width",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "(7, 7)",
+      "got": "(0, 0)",
+      "got_raw": "(0, 0)",
+      "exact_match": false,
+      "code_bleu": 0.32466791547509893,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "(8, 6)",
+      "got": "(10, 10)",
+      "got_raw": "(10, 10)",
+      "exact_match": false,
+      "code_bleu": 0.32466791547509893,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "captured.out",
+      "got": "captured.out",
+      "got_raw": "captured.out",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "\"1m\"",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "\"1h2m\"",
+      "got": "\"1h2m\"",
+      "got_raw": "\"1h2m\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "\"Valid Text\"",
+      "got": "\"Valid Text\"",
+      "got_raw": "\"Valid Text\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "ValidationError)",
+      "got": "ValidationError)",
+      "got_raw": "ValidationError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "'Expected format: a duration (e.g. 1m, 1s, 1h3m)' '(Ctrl-C to exit)\\n'",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "'msg'",
+      "got": "'msg'",
+      "got_raw": "'msg'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "ArgumentTypeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5517241379310345
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "'\\033[0;31mmsg\\033[0m'",
+      "got": "'\\\\033[31mmsg\\\\033[0m\\n'",
+      "got_raw": "'\\\\033[31mmsg\\\\033[0m\\n'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8695652173913043
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "'\\033[31;1merror\\033[0m'",
+      "got": "'error\\n'",
+      "got_raw": "'error\\n'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.48484848484848486
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "'\\033[0;33mdebug\\033[0m'",
+      "got": "'debug\\n'",
+      "got_raw": "'debug\\n'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.48484848484848486
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "'${color red}msg${color}'",
+      "got": "'  red: msg\\n'",
+      "got_raw": "'  red: msg\\n'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.46153846153846156
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "'${color red}error${color}'",
+      "got": "'error\\n'",
+      "got_raw": "'error\\n'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3888888888888889
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "'${color yellow}debug${color}'",
+      "got": "'debug\\n'",
+      "got_raw": "'debug\\n'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.358974358974359
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "cp.colors[color_name] + 'msg' + '\\033[0m'",
+      "got": "f'\\033[{COLOR_NAMES[color_name]}mmsg\\033[0m\\n'",
+      "got_raw": "f'\\033[{COLOR_NAMES[color_name]}mmsg\\033[0m\\n'",
+      "exact_match": false,
+      "code_bleu": 0.09036020036098448,
+      "edit_similarity": 0.5287356321839081
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "364",
+      "got": "364",
+      "got_raw": "364",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "timedelta(hours=1)",
+      "got": "timedelta(hours=1)",
+      "got_raw": "timedelta(hours=1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "timedelta(minutes=1)",
+      "got": "timedelta(1)",
+      "got_raw": "timedelta(1)",
+      "exact_match": false,
+      "code_bleu": 0.39640237166757364,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "\"Duration is invalid: junk\"",
+      "got": "'Invalid timedelta string: junk'",
+      "got_raw": "'Invalid timedelta string: junk'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4067796610169492
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "utils.get_times_from_duration(begin_1970_midnight, duration=120)",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "insanum/gcalcli",
+      "expected": "utils.get_times_from_duration( begin_1970_midnight, duration=\"2h\")",
+      "got": "(next_day, two_hrs_later)",
+      "got_raw": "(next_day, two_hrs_later)",
+      "exact_match": false,
+      "code_bleu": 0.16233395773754947,
+      "edit_similarity": 0.24175824175824176
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "5.0",
+      "got": "5.0",
+      "got_raw": "5.0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "6.0",
+      "got": "6.0",
+      "got_raw": "6.0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "10.0",
+      "got": "10.0",
+      "got_raw": "10.0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "N",
+      "got": "N",
+      "got_raw": "N",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "[0]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.39640237166757364,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "items",
+      "got": "items",
+      "got_raw": "items",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "flushes",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "max_size",
+      "got": "max_size",
+      "got_raw": "max_size",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "42",
+      "got": "42",
+      "got_raw": "42",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "ttl",
+      "got": "ttl",
+      "got_raw": "ttl",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "value1",
+      "got": "value1",
+      "got_raw": "value1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"value\"",
+      "got": "\"value\"",
+      "got_raw": "\"value\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"value_1\"",
+      "got": "\"value_1\"",
+      "got_raw": "\"value_1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "await cache()",
+      "got": "\"cached_value\"",
+      "got_raw": "\"cached_value\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.37037037037037035
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "CustomException)",
+      "got": "CustomException)",
+      "got_raw": "CustomException)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "N",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"sync\"",
+      "got": "\"test_value\"",
+      "got_raw": "\"test_value\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"async\"",
+      "got": "\"test_value\"",
+      "got_raw": "\"test_value\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3157894736842105
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"drain\"",
+      "got": "\"test_value\"",
+      "got_raw": "\"test_value\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3157894736842105
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"async_cm\"",
+      "got": "\"test_value\"",
+      "got_raw": "\"test_value\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "NotImplementedError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5161290322580645
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "N",
+      "got": "N",
+      "got_raw": "N",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "tally.max_active",
+      "got": "tally.count",
+      "got_raw": "tally.count",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.5925925925925926
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "2 * concurrency_limit",
+      "got": "2 * concurrency_limit",
+      "got_raw": "2 * concurrency_limit",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "{}",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "payload",
+      "got": "payload",
+      "got_raw": "payload",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "str(exc)",
+      "got": "str(exc)",
+      "got_raw": "str(exc)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "Exception)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "expected_policy",
+      "got": "expected_policy",
+      "got_raw": "expected_policy",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "expected_logged[\"pgqueuer\"]",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.12903225806451613
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "expected_logged[\"pgqueuer_log\"]",
+      "got": "expected_logged[\"pgqueuer_log\"]",
+      "got_raw": "expected_logged[\"pgqueuer_log\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "expected_logged[\"pgqueuer_schedules\"]",
+      "got": "expected_logged[\"pgqueuer_schedules\"]",
+      "got_raw": "expected_logged[\"pgqueuer_schedules\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "expected_logged[\"pgqueuer_statistics\"]",
+      "got": "expected_logged[\"pgqueuer_statistics\"]",
+      "got_raw": "expected_logged[\"pgqueuer_statistics\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "ValueError, match=\"'nonexistent' is not a valid Durability\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.30985915492957744
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "42",
+      "got": "42",
+      "got_raw": "42",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "TypeError)",
+      "got": "TypeError)",
+      "got_raw": "TypeError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"/fake/dir\"",
+      "got": "\"/fake/dir\"",
+      "got_raw": "\"/fake/dir\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "mock_factory",
+      "got": "mock_factory",
+      "got_raw": "mock_factory",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"sync_cm_value\"",
+      "got": "\"sync_cm_value\"",
+      "got_raw": "\"sync_cm_value\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"async_cm_value\"",
+      "got": "\"async_cm_value\"",
+      "got_raw": "\"async_cm_value\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"awaitable_value\"",
+      "got": "\"awaitable_value\"",
+      "got_raw": "\"awaitable_value\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "ImportError, match=\"No module named 'nonexistent_module'\")",
+      "got": "ImportError)",
+      "got_raw": "ImportError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.34285714285714286
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "AttributeError, match=\"Mock object has no attribute 'nonexistent_factory'\")",
+      "got": "AttributeError)",
+      "got_raw": "AttributeError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "{ \"object str can't be used in 'await' expression\", \"'str' object can't be awaited\", }",
+      "got": "\"Invalid input type\"",
+      "got_raw": "\"Invalid input type\"",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.22641509433962265
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0.1",
+      "got": "10.0",
+      "got_raw": "10.0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "3",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "t1",
+      "got": "t1",
+      "got_raw": "t1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "elapsed()",
+      "got": "0.0",
+      "got_raw": "0.0",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "timedelta(1)",
+      "got": "timedelta(1)",
+      "got_raw": "timedelta(1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "timedelta(2)",
+      "got": "timedelta(1)",
+      "got_raw": "timedelta(1)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.9166666666666666
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "timedelta(4)",
+      "got": "timedelta(4)",
+      "got_raw": "timedelta(4)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "timedelta(8)",
+      "got": "timedelta(8)",
+      "got_raw": "timedelta(8)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "2",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "n",
+      "got": "n",
+      "got_raw": "n",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "500",
+      "got": "1_000",
+      "got_raw": "1_000",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "5_000",
+      "got": "10_000",
+      "got_raw": "10_000",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7272727272727273
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "statuses",
+      "got": "statuses",
+      "got_raw": "statuses",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "sorted(order, reverse=True)",
+      "got": "[5, 10, 1]",
+      "got_raw": "[5, 10, 1]",
+      "exact_match": false,
+      "code_bleu": 0.162531757492419,
+      "edit_similarity": 0.10810810810810811
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"insert\"",
+      "got": "\"stable\"",
+      "got_raw": "\"stable\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"truncate\"",
+      "got": "\"TRUNCATE\"",
+      "got_raw": "\"TRUNCATE\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "event.root.id",
+      "got": "event.root.id",
+      "got_raw": "event.root.id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "NotImplementedError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5161290322580645
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"table_changed_event\"",
+      "got": "\"stable_changed_insert\"",
+      "got_raw": "\"stable_changed_insert\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8181818181818182
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "add_prefix(\"pgqueuer\")",
+      "got": "\"test_emit_stable_changed_insert\"",
+      "got_raw": "\"test_emit_stable_changed_insert\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.2545454545454545
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "(10, event.root.sent_at)",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.0660670619445956,
+      "edit_similarity": 0.15384615384615385
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "200",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "metrics_payload",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"text/plain; charset=utf-8\"",
+      "got": "\"text/plain;",
+      "got_raw": "\"text/plain;",
+      "exact_match": false,
+      "code_bleu": 0.2295748846661433,
+      "edit_similarity": 0.6153846153846154
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "'pgqueuer_queue_count{entrypoint=\"default\",status=\"pending\"} 3'",
+      "got": "\"pgqueuer_queue_count{entrypoint=default,status=pending} 3\"",
+      "got_raw": "\"pgqueuer_queue_count{entrypoint=default,status=pending} 3\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9344262295081968
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "[ 'custom_queue_count{aggregation=\"sum\",entrypoint=\"main\",status=\"queued\"} 2' ]",
+      "got": "[(\"custom_queue_count\", 2)]",
+      "got_raw": "[(\"custom_queue_count\", 2)]",
+      "exact_match": false,
+      "code_bleu": 0.20556680845025982,
+      "edit_similarity": 0.4528301886792453
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "[ 'custom_log_count{aggregation=\"sum\",entrypoint=\"main\",status=\"successful\"} 5' ]",
+      "got": "[ 'custom_log_count{aggregation=\"sum\",entrypoint=\"main\",status=\"successful\"} 5' ]",
+      "got_raw": "[ 'custom_log_count{aggregation=\"sum\",entrypoint=\"main\",status=\"successful\"} 5' ]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "N",
+      "got": "N",
+      "got_raw": "N",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "set(dir(Queries))",
+      "got": "set(Queries.__dict__)",
+      "got_raw": "set(Queries.__dict__)",
+      "exact_match": false,
+      "code_bleu": 0.31333304816101065,
+      "edit_similarity": 0.631578947368421
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "set(dir(SentryTracing))",
+      "got": "set(dir(SentryTracing))",
+      "got_raw": "set(dir(SentryTracing))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "set(dir(LogfireTracing))",
+      "got": "set(vars(LogfireTracing).values())",
+      "got_raw": "set(vars(LogfireTracing).values())",
+      "exact_match": false,
+      "code_bleu": 0.2770578117986203,
+      "edit_similarity": 0.7586206896551724
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "set(dir(InMemoryQueries))",
+      "got": "InMemoryQueries.__dict__",
+      "got_raw": "InMemoryQueries.__dict__",
+      "exact_match": false,
+      "code_bleu": 0.19362900077353815,
+      "edit_similarity": 0.6122448979591837
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "N",
+      "got": "N",
+      "got_raw": "N",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"job_fetch\"",
+      "got": "\"test\"",
+      "got_raw": "\"test\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.35294117647058826
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"pick_local\"",
+      "got": "\"test\"",
+      "got_raw": "\"test\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "list(range(N))",
+      "got": "list(range(N))",
+      "got_raw": "list(range(N))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "[\"to_be_picked\"] * N",
+      "got": "[\"to_be_picked\"]",
+      "got_raw": "[\"to_be_picked\"]",
+      "exact_match": false,
+      "code_bleu": 0.5045378491522287,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "max_concurrent_tasks",
+      "got": "N",
+      "got_raw": "N",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "N",
+      "got": "N",
+      "got_raw": "N",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "backoff.start_delay",
+      "got": "timedelta(milliseconds=1)",
+      "got_raw": "timedelta(milliseconds=1)",
+      "exact_match": false,
+      "code_bleu": 0.18575057999133598,
+      "edit_similarity": 0.22727272727272727
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "timedelta(milliseconds=2)",
+      "got": "timedelta(milliseconds=1)",
+      "got_raw": "timedelta(milliseconds=1)",
+      "exact_match": false,
+      "code_bleu": 0.6434588841607617,
+      "edit_similarity": 0.96
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "[timedelta(milliseconds=2)]",
+      "got": "[timedelta(milliseconds=2)]",
+      "got_raw": "[timedelta(milliseconds=2)]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "timedelta(milliseconds=2).total_seconds())",
+      "got": "1)]",
+      "got_raw": "1)]",
+      "exact_match": false,
+      "code_bleu": 0.12321845503770609,
+      "edit_similarity": 0.044444444444444446
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "[pytest.approx(timedelta(milliseconds=2).total_seconds())]",
+      "got": "[1]",
+      "got_raw": "[1]",
+      "exact_match": false,
+      "code_bleu": 0.08823529411764705,
+      "edit_similarity": 0.06557377049180328
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "N",
+      "got": "N",
+      "got_raw": "N",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "v[1]",
+      "got": "v[1]",
+      "got_raw": "v[1]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "v[0]",
+      "got": "v[0]",
+      "got_raw": "v[0]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "job_id",
+      "got": "job_id",
+      "got_raw": "job_id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"picked\"",
+      "got": "\"picked\"",
+      "got_raw": "\"picked\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "len(jobs)",
+      "got": "len(jobs)",
+      "got_raw": "len(jobs)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "entrypoint",
+      "got": "entrypoint",
+      "got_raw": "entrypoint",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "expression",
+      "got": "expression",
+      "got_raw": "expression",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"1 * * * *\"",
+      "got": "\"1 * * * *\"",
+      "got_raw": "\"1 * * * *\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"2 * * * *\"",
+      "got": "\"2 * * * *\"",
+      "got_raw": "\"2 * * * *\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"3 * * * *\"",
+      "got": "\"2 * * * *\"",
+      "got_raw": "\"2 * * * *\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"sample_task\"",
+      "got": "\"sample_task\"",
+      "got_raw": "\"sample_task\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "n_tasks",
+      "got": "n_tasks",
+      "got_raw": "n_tasks",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "RuntimeError)",
+      "got": "RuntimeError)",
+      "got_raw": "RuntimeError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "shutdown_event",
+      "got": "shutdown_event",
+      "got_raw": "shutdown_event",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "signal.SIGINT)",
+      "got": "ANY)",
+      "got_raw": "ANY)",
+      "exact_match": false,
+      "code_bleu": 0.2686424829558855,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "signal.SIGTERM)",
+      "got": "signal.SIGTERM)",
+      "got_raw": "signal.SIGTERM)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "caplog.text.lower()",
+      "got": "caplog.text",
+      "got_raw": "caplog.text",
+      "exact_match": false,
+      "code_bleu": 0.36038417796587763,
+      "edit_similarity": 0.7333333333333333
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "FailingListenerError)",
+      "got": "FailingListenerError)",
+      "got_raw": "FailingListenerError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "Exception, match=\"Simulated failure\")",
+      "got": "Exception)",
+      "got_raw": "Exception)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.425531914893617
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "ValueError, match=\"'restart_delay' must be >= 0\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.36666666666666664
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "NotImplementedError, match=\"Unsupported instance type: .*InvalidManager.*\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.18604651162790697
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "NotImplementedError, match=r\"Unsupported instance type: .*InvalidManager.*\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.153509990260506,
+      "edit_similarity": 0.1839080459770115
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "RuntimeError, match=r\"max_concurrent_tasks must be at least twice the batch size\")",
+      "got": "queue_manager",
+      "got_raw": " queue_manager",
+      "exact_match": false,
+      "code_bleu": 0.07142857142857142,
+      "edit_similarity": 0.14736842105263157
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "N",
+      "got": "N",
+      "got_raw": "N",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "[True] * N",
+      "got": "[True] * N",
+      "got_raw": "[True] * N",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "N",
+      "got": "N",
+      "got_raw": "N",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "N",
+      "got": "N",
+      "got_raw": "N",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "5",
+      "got": "1.0",
+      "got_raw": "1.0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "now",
+      "got": "now",
+      "got_raw": "now",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "job_id",
+      "got": "job_id",
+      "got_raw": "job_id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "additional",
+      "got": "additional",
+      "got_raw": "additional",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "\"ValueError\"",
+      "got": "ValueError",
+      "got_raw": "ValueError",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "expected_keys",
+      "got": "expected_keys",
+      "got_raw": "expected_keys",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "record.exception_message",
+      "got": "str(exc)",
+      "got_raw": "str(exc)",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "janbjorge/pgqueuer",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "expected_vals",
+      "got": "expected_vals",
+      "got_raw": "expected_vals",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "n_expected_vals",
+      "got": "n_expected_vals",
+      "got_raw": "n_expected_vals",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "n_expected_returns",
+      "got": "n_expected_returns",
+      "got_raw": "n_expected_returns",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "sorted(set(ret_list))",
+      "got": "ret_list",
+      "got_raw": "ret_list",
+      "exact_match": false,
+      "code_bleu": 0.08494336535733721,
+      "edit_similarity": 0.5517241379310345
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "\"a-project\"",
+      "got": "'b-project'",
+      "got_raw": "'b-project'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7272727272727273
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "\"b-project\"",
+      "got": "\"b-project\"",
+      "got_raw": "\"b-project\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "''",
+      "got": "''",
+      "got_raw": "''",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "42",
+      "got": "42",
+      "got_raw": "42",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "23",
+      "got": "23",
+      "got_raw": "23",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "'42'",
+      "got": "42",
+      "got_raw": "42",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "3.14",
+      "got": "3.14",
+      "got_raw": "3.14",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "aniceday_lastMoon",
+      "got": "aniceday_lastMoon",
+      "got_raw": "aniceday_lastMoon",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "thenextday_lastMoon",
+      "got": "thenextday_lastMoon",
+      "got_raw": "thenextday_lastMoon",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "''",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "'A'",
+      "got": "'A'",
+      "got_raw": "'A'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "'B'",
+      "got": "'B'",
+      "got_raw": "'B'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "'[]'",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "keys",
+      "got": "keys",
+      "got_raw": "keys",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "'foo'",
+      "got": "'foo'",
+      "got_raw": "'foo'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "result",
+      "got": "result",
+      "got_raw": "result",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "header",
+      "got": "header",
+      "got_raw": "header",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "'A, B'",
+      "got": "['A', 'B']",
+      "got_raw": "['A', 'B']",
+      "exact_match": false,
+      "code_bleu": 0.2295748846661433,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "Abort)",
+      "got": "Abort)",
+      "got_raw": "Abort)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "jazzband/Watson",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "atol)",
+      "got": "atol=atol)",
+      "got_raw": "atol=atol)",
+      "exact_match": false,
+      "code_bleu": 0.47287080450158786,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "kapre_again)",
+      "got": "kapre_again)",
+      "got_raw": "kapre_again)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "RuntimeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "input_shape[axis])",
+      "got": "input_shape[axis])",
+      "got_raw": "input_shape[axis])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "NotImplementedError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5161290322580645
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "spec_augmented[0].shape)",
+      "got": "input_shape[1:])",
+      "got_raw": "input_shape[1:])",
+      "exact_match": false,
+      "code_bleu": 0.2251637052363564,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "mel_fb)",
+      "got": "mel_fb)",
+      "got_raw": "mel_fb)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "mu_src_recon)",
+      "got": "mu_src_recon)",
+      "got_raw": "mu_src_recon)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "x_decibel_ref)",
+      "got": "x_decibel_ref)",
+      "got_raw": "x_decibel_ref)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "(n_freq, n_mels)",
+      "got": "mel_fb_ref.shape",
+      "got_raw": "mel_fb_ref.shape",
+      "exact_match": false,
+      "code_bleu": 0.22795070569547776,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "(n_freq, n_bins)",
+      "got": "(n_bins, n_freq)",
+      "got_raw": "(n_bins, n_freq)",
+      "exact_match": false,
+      "code_bleu": 0.35930411196308426,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "KPB._get_floatx()",
+      "got": "mel_fb_ref.dtype",
+      "got_raw": "mel_fb_ref.dtype",
+      "exact_match": false,
+      "code_bleu": 0.2710806010829534,
+      "edit_similarity": 0.24242424242424243
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "mu_src_kapre.numpy())",
+      "got": "mu_src_kapre)",
+      "got_raw": "mu_src_kapre)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.7647058823529411
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "frames_ref)",
+      "got": "frames_ref)",
+      "got_raw": "frames_ref)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "energies_ref)",
+      "got": "energies_ref)",
+      "got_raw": "energies_ref)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "mfcc_kapre[1:])",
+      "got": "mfcc_kapre[1:])",
+      "got_raw": "mfcc_kapre[1:])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "mfcc_kapre[0] / np.sqrt(2.0))",
+      "got": "mfcc_kapre[0])",
+      "got_raw": "mfcc_kapre[0])",
+      "exact_match": false,
+      "code_bleu": 0.3313628036529378,
+      "edit_similarity": 0.6511627906976745
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "S)",
+      "got": "np.abs(S))",
+      "got_raw": "np.abs(S))",
+      "exact_match": false,
+      "code_bleu": 0.24446151121745047,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "S_lite)",
+      "got": "S_lite)",
+      "got_raw": "S_lite)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "S_phase)",
+      "got": "S_phase)",
+      "got_raw": "S_phase)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "np.sin(b))",
+      "got": "np.sin(b))",
+      "got_raw": "np.sin(b))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "np.cos(b))",
+      "got": "np.cos(b))",
+      "got_raw": "np.cos(b))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "np.abs(b))",
+      "got": "np.abs(b))",
+      "got_raw": "np.abs(b))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "delta_ref)",
+      "got": "delta_ref)",
+      "got_raw": "delta_ref)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "np.real(b))",
+      "got": "np.real(b))",
+      "got_raw": "np.real(b))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "np.imag(b))",
+      "got": "np.imag(b))",
+      "got_raw": "np.imag(b))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "np.shape(b))",
+      "got": "np.shape(b))",
+      "got_raw": "np.shape(b))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "recon_S.shape)",
+      "got": "recon_S.shape)",
+      "got_raw": "recon_S.shape)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "S_approx_phase)",
+      "got": "S_approx_phase)",
+      "got_raw": "S_approx_phase)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "recon_waveform)",
+      "got": "recon_waveform)",
+      "got_raw": "recon_waveform)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "x_concat.shape)",
+      "got": "x_concat.shape)",
+      "got_raw": "x_concat.shape)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "other_channels)",
+      "got": "other_channels)",
+      "got_raw": "other_channels)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "mag_phase_ref.shape)",
+      "got": "mag_phase_ref.shape)",
+      "got_raw": "mag_phase_ref.shape)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "keunwoochoi/kapre",
+      "expected": "acceptable_fail_ratio",
+      "got": "acceptable_fail_ratio",
+      "got_raw": "acceptable_fail_ratio * len(a))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "deprecated.__doc__",
+      "got": "deprecated.__doc__",
+      "got_raw": "deprecated.__doc__",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "version.__class__.__name__",
+      "got": "version",
+      "got_raw": "version",
+      "exact_match": false,
+      "code_bleu": 0.11892071150027211,
+      "edit_similarity": 0.42424242424242425
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "__file__",
+      "got": "__file__",
+      "got_raw": "__file__",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "demo_lineno + 1",
+      "got": "demo_lineno",
+      "got_raw": "demo_lineno",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.8461538461538461
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "str(warn.message)",
+      "got": "str(warn)",
+      "got_raw": "str(warn)",
+      "exact_match": false,
+      "code_bleu": 0.4714045207910316,
+      "edit_similarity": 0.6923076923076923
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "use_foo_lineno + 1",
+      "got": "use_foo_lineno",
+      "got_raw": "use_foo_lineno",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.875
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "u\"I am deprecated!\"",
+      "got": "u\"I am deprecated!\"",
+      "got_raw": "u\"I am deprecated!\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "3.14",
+      "got": "3.14",
+      "got_raw": "3.14",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "obj1",
+      "got": "obj1",
+      "got_raw": "obj1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "\"five\"",
+      "got": "\"five\"",
+      "got_raw": "\"five\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "\"\\n\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "actual",
+      "got": "actual",
+      "got_raw": "actual",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "str(warn.message)",
+      "got": "str(warn)",
+      "got_raw": "str(warn)",
+      "exact_match": false,
+      "code_bleu": 0.4714045207910316,
+      "edit_similarity": 0.6923076923076923
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "u\"I am deprecated!\"",
+      "got": "\"I am deprecated!\"",
+      "got_raw": "\"I am deprecated!\"",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.972972972972973
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "3.14",
+      "got": "3.14",
+      "got_raw": "3.14",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "obj1",
+      "got": "obj1",
+      "got_raw": "obj1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "\"five\"",
+      "got": "\"five\"",
+      "got_raw": "\"five\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "laurent-laporte-pro/deprecated",
+      "expected": "[ {\"category\": V2DeprecationWarning, \"message\": \"epsilon is deprecated in version v2\"}, {\"category\": V2DeprecationWarning, \"message\": \"start is removed in version v2\"}, {\"category\": DeprecationWarning, \"message\": \"epsilon is deprecated in version v1.1\"}, ]",
+      "got": "[{\"message\": \"The `epsilon` parameter is deprecated and will be removed in a future version. Use `n` instead.\", \"category\": UserWarning}]",
+      "got_raw": "[{\"message\": \"The `epsilon` parameter is deprecated and will be removed in a future version. Use `n` instead.\", \"category\": UserWarning}]",
+      "exact_match": false,
+      "code_bleu": 0.08280443429793394,
+      "edit_similarity": 0.07633587786259542
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "repr(e)",
+      "got": "str(e)",
+      "got_raw": "str(e)",
+      "exact_match": false,
+      "code_bleu": 0.668740304976422,
+      "edit_similarity": 0.6153846153846154
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "ma.keys()",
+      "got": "ma",
+      "got_raw": "ma",
+      "exact_match": false,
+      "code_bleu": 0.11892071150027211,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "ma.items()",
+      "got": "ma.items()",
+      "got_raw": "ma.items()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "len(tensors)",
+      "got": "len(all_tensors)",
+      "got_raw": "len(all_tensors)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "len(reference)",
+      "got": "num_parts",
+      "got_raw": "num_parts",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.17391304347826086
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "list(range(10))",
+      "got": "[10]",
+      "got_raw": "[10]",
+      "exact_match": false,
+      "code_bleu": 0.19362900077353815,
+      "edit_similarity": 0.21052631578947367
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "str(excinfo.value)",
+      "got": "str(excinfo.value)",
+      "got_raw": "str(excinfo.value)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "len(reference_knn)",
+      "got": "len(reference_knn)",
+      "got_raw": "len(reference_knn)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "dict(name=\"expert1\")",
+      "got": "{",
+      "got_raw": "{ \"name\": \"expert1\", \"k\": 1, }",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "dict(name=\"expert2\")",
+      "got": "dict(name=\"expert2\")",
+      "got_raw": "dict(name=\"expert2\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "len(targets_for_peer)",
+      "got": "len(targets_for_peer)",
+      "got_raw": "len(targets_for_peer)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "len(all_active_neighbors)",
+      "got": "len(all_active_neighbors)",
+      "got_raw": "len(all_active_neighbors)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "len(routing_table.buckets)",
+      "got": "len(bucket)",
+      "got_raw": "len(bucket)",
+      "exact_match": false,
+      "code_bleu": 0.3387551654364098,
+      "edit_similarity": 0.5945945945945946
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "DHTID.xor_distance(id2, id2)",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.0625,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "test_pb2.TestResponse(number=45)",
+      "got": "45",
+      "got_raw": "45",
+      "exact_match": false,
+      "code_bleu": 0.07432544468767006,
+      "edit_similarity": 0.11764705882352941
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "test_pb2.TestResponse(number=11)",
+      "got": "test_pb2.TestResponse(number=11)",
+      "got_raw": "test_pb2.TestResponse(number=11)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "test_pb2.TestResponse(number=100)",
+      "got": "100",
+      "got_raw": "100",
+      "exact_match": false,
+      "code_bleu": 0.07432544468767006,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "await protocol.call_ping(peer1_id)",
+      "got": "protocol.node_id",
+      "got_raw": "protocol.node_id",
+      "exact_match": false,
+      "code_bleu": 0.2548300960720116,
+      "edit_similarity": 0.56
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "await protocol.call_ping(peer_peer_id)",
+      "got": "protocol.node_id",
+      "got_raw": "protocol.node_id",
+      "exact_match": false,
+      "code_bleu": 0.2548300960720116,
+      "edit_similarity": 0.5185185185185185
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "test_pb2.TestResponse(number=(i // 2) ** 2)",
+      "got": "test_pb2.TestRequest(number=i)",
+      "got_raw": "test_pb2.TestRequest(number=i)",
+      "exact_match": false,
+      "code_bleu": 0.1907082808182838,
+      "edit_similarity": 0.7123287671232876
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "test_pb2.TestResponse(number=(i // 2) ** 3)",
+      "got": "test_pb2.TestResponse(number=(i // 2) ** 3)",
+      "got_raw": "test_pb2.TestResponse(number=(i // 2) ** 3)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "max((len(serialized_tensor.buffer) - 1) // chunk_size + 1, 1)",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.02973017787506803,
+      "edit_similarity": 0.03225806451612903
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "x**2",
+      "got": "x**2",
+      "got_raw": "x**2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "emas[2]",
+      "got": "emas[2]",
+      "got_raw": "emas[2]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "tensor.dtype",
+      "got": "tensor.dtype",
+      "got_raw": "tensor.dtype",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "stored.value",
+      "got": "stored",
+      "got_raw": "stored",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "record.value",
+      "got": "record",
+      "got_raw": "record",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "wall_time / 3",
+      "got": "0.0",
+      "got_raw": "0.0",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "control2.stage",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "p2ps[-1].peer_id",
+      "got": "p2ps[-1].peer_id",
+      "got_raw": "p2ps[-1].peer_id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "plain_record.value",
+      "got": "plain_record",
+      "got_raw": "plain_record",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "other_peer.peer_id",
+      "got": "first_peer.peer_id",
+      "got_raw": "first_peer.peer_id",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.7777777777777778
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "signed_record.value",
+      "got": "signed_record.value",
+      "got_raw": "signed_record.value",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "step_time_deltas[i]",
+      "got": "emas[i]",
+      "got_raw": "emas[i]",
+      "exact_match": false,
+      "code_bleu": 0.668740304976422,
+      "edit_similarity": 0.5384615384615384
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "tensor.requires_grad",
+      "got": "tensor.requires_grad",
+      "got_raw": "tensor.requires_grad",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "n_processes * n_coros",
+      "got": "n_coros",
+      "got_raw": "n_coros",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "server_info_2.peer_id",
+      "got": "server_info_2.peer_id",
+      "got_raw": "server_info_2.peer_id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "server_info_3.peer_id",
+      "got": "server_info_3.peer_id",
+      "got_raw": "server_info_3.peer_id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "remaining_peer1.peer_id",
+      "got": "remaining_peer1.peer_id",
+      "got_raw": "remaining_peer1.peer_id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "EXPERT_WEIGHT_UPDATES + 1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.07692307692307693
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "asyncio.IncompleteReadError)",
+      "got": "Exception)",
+      "got_raw": "Exception)",
+      "exact_match": false,
+      "code_bleu": 0.2686424829558855,
+      "edit_similarity": 0.2631578947368421
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "optimizer.tracker.global_epoch",
+      "got": "total_epochs",
+      "got_raw": "total_epochs",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.47619047619047616
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "BACKWARD_PASSES_BEFORE_SAVE + 1",
+      "got": "BACKWARD_PASSES_BEFORE_SAVE",
+      "got_raw": "BACKWARD_PASSES_BEFORE_SAVE",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.9310344827586207
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "total_samples_accumulated.value",
+      "got": "total_samples_accumulated.value",
+      "got_raw": "total_samples_accumulated.value",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "concurrent.futures.TimeoutError)",
+      "got": "P2PDaemonError",
+      "got_raw": "P2PDaemonError, match=r\"(?i)Failed to connect to bootstrap peers\")",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.30434782608695654
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "third_validator.local_public_key",
+      "got": "third_validator.local_public_key",
+      "got_raw": "third_validator.local_public_key",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "second_validator.local_public_key",
+      "got": "second_validator.local_public_key",
+      "got_raw": "second_validator.local_public_key",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "2 if sync_epoch_when_averaging else 1",
+      "got": "2 + 1)",
+      "got_raw": "2 + 1)",
+      "exact_match": false,
+      "code_bleu": 0.3181082915068203,
+      "edit_similarity": 0.18604651162790697
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "4",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "2",
+      "got": "self.k",
+      "got_raw": "self.k",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "4",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "7",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "0",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "2",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "13",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "16",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "32",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "0.5",
+      "got": "0.9",
+      "got_raw": "0.9",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "123",
+      "got": "123",
+      "got_raw": "123",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "456",
+      "got": "456",
+      "got_raw": "456",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "124",
+      "got": "124",
+      "got_raw": "124",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "125",
+      "got": "125",
+      "got_raw": "125",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "126",
+      "got": "126",
+      "got_raw": "126",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "-99",
+      "got": "127",
+      "got_raw": "127",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "777",
+      "got": "777",
+      "got_raw": "777",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "999",
+      "got": "999",
+      "got_raw": "999",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "0.0",
+      "got": "PEAK_LR",
+      "got_raw": "PEAK_LR",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "0.2",
+      "got": "0.2",
+      "got_raw": "0.2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "150",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "321",
+      "got": "321",
+      "got_raw": "321",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "700",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "100",
+      "got": "100",
+      "got_raw": "100",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "\"\"",
+      "got": "\"/ip4/127.0.0.1/udp/5678\"",
+      "got_raw": "\"/ip4/127.0.0.1/udp/5678\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.14814814814814814
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "\"bob\"",
+      "got": "\"bob\"",
+      "got_raw": "\"bob\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "\"pew\"",
+      "got": "\"pew\"",
+      "got_raw": "\"pew\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "\"ip4\"",
+      "got": "\"ip4\"",
+      "got_raw": "\"ip4\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "\"123\"",
+      "got": "proto",
+      "got_raw": "proto",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "b\"val\"",
+      "got": "b\"val\"",
+      "got_raw": "b\"val\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "b\"pong\"",
+      "got": "b\"pong\"",
+      "got_raw": "b\"pong\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "b\"ping\"",
+      "got": "node_id",
+      "got_raw": "node_id",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.14285714285714285
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "\"alice\"",
+      "got": "\"test\"",
+      "got_raw": "\"test\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.46153846153846156
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "b\"val1\"",
+      "got": "b\"val1\"",
+      "got_raw": "b\"val1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "b\"val2\"",
+      "got": "b\"val2\"",
+      "got_raw": "b\"val2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "\"5555\")",
+      "got": "5555)",
+      "got_raw": "5555)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.8333333333333334
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "b\"\\x05\"",
+      "got": "b\"\\x05\"",
+      "got_raw": "b\"\\x05\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "b\"okay\"",
+      "got": "b\"okay\"",
+      "got_raw": "b\"okay\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "\"value1\"",
+      "got": "\"value1\"",
+      "got_raw": "\"value1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "b\"value\"",
+      "got": "record.value",
+      "got_raw": "record.value",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "b\"val1a2\"",
+      "got": "b\"val1a2\"",
+      "got_raw": "b\"val1a2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "b\"foo_bar\"",
+      "got": "b\"foo_bar\"",
+      "got_raw": "b\"foo_bar\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "b\"true_value\"",
+      "got": "b\"true_value\"",
+      "got_raw": "b\"true_value\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "\"string_value\"",
+      "got": "42",
+      "got_raw": "42",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "None",
+      "got": "proto3",
+      "got_raw": "proto3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "d",
+      "got": "d",
+      "got_raw": "d",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "m2",
+      "got": "m2",
+      "got_raw": "m2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "m1",
+      "got": "m3",
+      "got_raw": "m3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "m3",
+      "got": "m3",
+      "got_raw": "m3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "ma",
+      "got": "ma",
+      "got_raw": "ma",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "buf",
+      "got": "buf",
+      "got_raw": "buf",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "ref",
+      "got": "ref",
+      "got_raw": "ref",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "beta",
+      "got": "beta",
+      "got_raw": "beta",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "ours",
+      "got": "ours",
+      "got_raw": "ours",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "line",
+      "got": "line",
+      "got_raw": "line",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "alpha",
+      "got": "alpha",
+      "got_raw": "alpha",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "value",
+      "got": "value",
+      "got_raw": "value",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "proto",
+      "got": "proto",
+      "got_raw": "proto",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "MADDR",
+      "got": "MADDR",
+      "got_raw": "MADDR",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "proto",
+      "got": "proto",
+      "got_raw": "proto",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "result",
+      "got": "result",
+      "got_raw": "result",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "proto2",
+      "got": "proto2",
+      "got_raw": "proto2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "proto3",
+      "got": "proto3",
+      "got_raw": "proto3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "proto4",
+      "got": "proto4",
+      "got_raw": "proto4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "string",
+      "got": "string",
+      "got_raw": "string",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "result",
+      "got": "result",
+      "got_raw": "result",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "PEAK_LR",
+      "got": "PEAK_LR",
+      "got_raw": "PEAK_LR",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "integer",
+      "got": "integer",
+      "got_raw": "integer",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "PEER_ID",
+      "got": "PEER_ID",
+      "got_raw": "PEER_ID",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "n_peers",
+      "got": "n_peers",
+      "got_raw": "n_peers",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "phony_id",
+      "got": "phony_id",
+      "got_raw": "phony_id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "query_id",
+      "got": "query_id",
+      "got_raw": "query_id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "beam_size",
+      "got": "beam_size",
+      "got_raw": "beam_size",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "ip6_bytes",
+      "got": "ip6_bytes",
+      "got_raw": "ip6_bytes",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "num_added",
+      "got": "len(set(all_active_neighbors))",
+      "got_raw": "len(set(all_active_neighbors))",
+      "exact_match": false,
+      "code_bleu": 0.1561969968460128,
+      "edit_similarity": 0.20512820512820512
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "threshold",
+      "got": "threshold",
+      "got_raw": "threshold",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "total_size",
+      "got": "total_size",
+      "got_raw": "total_size",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "ip6_string",
+      "got": "ip6_string",
+      "got_raw": "ip6_string",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "TypeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "Exception)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "table_size",
+      "got": "len(routing_table.buckets)",
+      "got_raw": "len(routing_table.buckets)",
+      "exact_match": false,
+      "code_bleu": 0.18575057999133598,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "second_line",
+      "got": "second_line",
+      "got_raw": "second_line",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "expected_key",
+      "got": "expected_key",
+      "got_raw": "expected_key",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "RuntimeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "routing_table",
+      "got": "routing_table",
+      "got_raw": "routing_table",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "P2PDaemonError)",
+      "got": "P2PHandlerError)",
+      "got_raw": "P2PHandlerError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.7096774193548387
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "first_report_msg",
+      "got": "first_report_msg",
+      "got_raw": "first_report_msg",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "P2PHandlerError)",
+      "got": "P2PHandlerError)",
+      "got_raw": "P2PHandlerError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "allow_extra_keys",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "{}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "(4, 3)",
+      "got": "(4, 5)",
+      "got_raw": "(4, 5)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.8333333333333334
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "[MADDR]",
+      "got": "[MADDR]",
+      "got_raw": "[MADDR]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "(50, 50)",
+      "got": "(50, 50)",
+      "got_raw": "(50, 50)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "(1, 0, 1)",
+      "got": "(0, 1, 1)",
+      "got_raw": "(0, 1, 1)",
+      "exact_match": false,
+      "code_bleu": 0.5169731539571706,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "(1, 0, 0)",
+      "got": "(0, 0, 0)",
+      "got_raw": "(0, 0, 0)",
+      "exact_match": false,
+      "code_bleu": 0.7071067811865475,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "(41, 19, 0)",
+      "got": "[10, 10, 10]",
+      "got_raw": "[10, 10, 10]",
+      "exact_match": false,
+      "code_bleu": 0.20556680845025982,
+      "edit_similarity": 0.6086956521739131
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "(1, hidden_dim)",
+      "got": "(1, hidden_dim)",
+      "got_raw": "(1, hidden_dim)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "(3, hidden_dim)",
+      "got": "(3, hidden_dim)",
+      "got_raw": "(3, hidden_dim)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "(4, 3, hidden_dim)",
+      "got": "(4, 1)",
+      "got_raw": "(4, 1)",
+      "exact_match": false,
+      "code_bleu": 0.383774975651265,
+      "edit_similarity": 0.4166666666666667
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "(1 / num_experts) / 2",
+      "got": "0.0",
+      "got_raw": "0.0",
+      "exact_match": false,
+      "code_bleu": 0.07142857142857142,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "(P2PDaemonError, ConnectionError))",
+      "got": "P2PDaemonError)",
+      "got_raw": "P2PDaemonError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.6122448979591837
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "[SchemaValidator, RSASignatureValidator]",
+      "got": "[RSASignatureValidator, SchemaValidator]",
+      "got_raw": "[RSASignatureValidator, SchemaValidator]",
+      "exact_match": false,
+      "code_bleu": 0.35930411196308426,
+      "edit_similarity": 0.575
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "(protocol.serializer.dumps(value1), expiration)",
+      "got": "value1",
+      "got_raw": "value1",
+      "exact_match": false,
+      "code_bleu": 0.049550296458446705,
+      "edit_similarity": 0.22641509433962265
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "True",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "learning-at-home/hivemind",
+      "expected": "False",
+      "got": "b\"value4\"",
+      "got_raw": "b\"value4\", time + 4)",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.42857142857142855
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "200",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "\"utf-8\"",
+      "got": "\"utf-8\"",
+      "got_raw": "\"utf-8\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "res.text",
+      "got": "res.text",
+      "got_raw": "res.text",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "\"litmus suite failed: check the log\")",
+      "got": "\"litmus failed\")",
+      "got_raw": "\"litmus failed\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6037735849056604
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "url",
+      "got": "url",
+      "got_raw": "url",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "self.root",
+      "got": "self.root",
+      "got_raw": "self.root",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "__debug__)",
+      "got": ".lm._verbose",
+      "got_raw": ".lm._verbose = 0",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.2727272727272727
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "AssertionError)",
+      "got": "lm)",
+      "got_raw": "lm)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.1111111111111111
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "__debug__)",
+      "got": "pm)",
+      "got_raw": "pm)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.15384615384615385
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "AssertionError)",
+      "got": "pm)",
+      "got_raw": "pm)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.1111111111111111
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "\"my name is joe\"",
+      "got": "\"my name is joe\"",
+      "got_raw": "\"my name is joe\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "data1",
+      "got": "data1",
+      "got_raw": "data1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "1735660800",
+      "got": "1735660800",
+      "got_raw": "1735660800",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "__debug__)",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.14285714285714285
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "3)",
+      "got": "3)",
+      "got_raw": "3)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "5)",
+      "got": "5)",
+      "got_raw": "5)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "0)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "42)",
+      "got": "42)",
+      "got_raw": "42)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "32)",
+      "got": "26)",
+      "got_raw": "26)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "204)",
+      "got": "201)",
+      "got_raw": "201)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "self.SIZE)",
+      "got": "self.SIZE)",
+      "got_raw": "self.SIZE)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "\"further read() returns ''\")",
+      "got": "\"empty read returns ''\")",
+      "got_raw": "\"empty read returns ''\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.7692307692307693
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "\"Read after close() returns ''\")",
+      "got": "\"read after close should return \"\"\")",
+      "got_raw": "\"read after close should return \"\"\")",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.7647058823529411
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "\"last chunk delivers the reminder\")",
+      "got": "\"Read after close() returns ''\")",
+      "got_raw": "\"Read after close() returns ''\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.3582089552238806
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "[]",
+      "got": "()",
+      "got_raw": "()",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "baseOutput",
+      "got": "baseOutput",
+      "got_raw": "baseOutput",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "__debug__)",
+      "got": "None)",
+      "got_raw": "None)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.26666666666666666
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "res",
+      "got": "res.body",
+      "got_raw": "res.body",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.5454545454545454
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "data",
+      "got": "data",
+      "got_raw": "data",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "data1",
+      "got": "data1",
+      "got_raw": "data1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "data2",
+      "got": "data2",
+      "got_raw": "data2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "data3",
+      "got": "data3",
+      "got_raw": "data3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "__debug__)",
+      "got": ".app",
+      "got_raw": ".app = webtest.TestApp({})",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "mar10/wsgidav",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "llm",
+      "got": "llm",
+      "got_raw": "llm",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "llm",
+      "got": "llm",
+      "got_raw": "llm",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "val",
+      "got": "val",
+      "got_raw": "val",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "body",
+      "got": "body",
+      "got_raw": "body",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "output",
+      "got": "output",
+      "got_raw": "output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "output",
+      "got": "output",
+      "got_raw": "output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "output",
+      "got": "output",
+      "got_raw": "output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "fake_llm",
+      "got": "fake_llm",
+      "got_raw": "fake_llm",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "EXAMPLES",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "template",
+      "got": "template",
+      "got_raw": "template",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "KeyError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.7
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "loaded_llm",
+      "got": "loaded_llm",
+      "got_raw": "loaded_llm",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError",
+      "got_raw": "ValueError, match=\".*text.*\")",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError",
+      "got_raw": "ValueError, match=\"text must be a string\")",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "loaded_llm)",
+      "got": "loaded_llm)",
+      "got_raw": "loaded_llm)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "loaded_llm)",
+      "got": "loaded_llm)",
+      "got_raw": "loaded_llm)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "loaded_llm)",
+      "got": "loaded_llm)",
+      "got_raw": "loaded_llm)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "loaded_llm)",
+      "got": "loaded_llm)",
+      "got_raw": "loaded_llm)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "loaded_llm)",
+      "got": "loaded_llm)",
+      "got_raw": "loaded_llm)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError",
+      "got_raw": "ValueError, match=\"foo\")",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError",
+      "got_raw": "ValueError, match=\"foo\")",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError",
+      "got_raw": "ValueError, match=\"args\")",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError",
+      "got_raw": "ValueError, match=\".*must be a positive integer.*\")",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError",
+      "got_raw": "ValueError, match=\"No action input found\")",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError",
+      "got_raw": "ValueError, match=\"Key not found in input\")",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError",
+      "got_raw": "ValueError, match=\"Missing input variables\")",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError",
+      "got_raw": "ValueError, match=\"Input variables must be provided\")",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "ValueError)",
+      "got": "ValueError",
+      "got_raw": "ValueError, match=\"Input variables must be\")",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "second_output",
+      "got": "second_output",
+      "got_raw": "second_output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "second_output",
+      "got": "second_output",
+      "got_raw": "second_output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "second_output",
+      "got": "second_output",
+      "got_raw": "second_output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "RuntimeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "expected_docs",
+      "got": "expected_docs",
+      "got_raw": "expected_docs",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "simple_prompt",
+      "got": "simple_prompt",
+      "got_raw": "simple_prompt",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "fake_llm_chain",
+      "got": "fake_llm_chain",
+      "got_raw": "fake_llm_chain",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "expected_output",
+      "got": "expected_output",
+      "got_raw": "expected_output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "expected_output",
+      "got": "expected_output",
+      "got_raw": "expected_output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "expected_output",
+      "got": "expected_output",
+      "got_raw": "expected_output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "expected_output",
+      "got": "expected_output",
+      "got_raw": "expected_output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "expected_output",
+      "got": "expected_output",
+      "got_raw": "expected_output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "expected_output",
+      "got": "expected_output",
+      "got_raw": "expected_output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "expected_output",
+      "got": "expected_output",
+      "got_raw": "expected_output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "expected_output",
+      "got": "expected_output",
+      "got_raw": "expected_output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "AssertionError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5384615384615384
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "expected_result",
+      "got": "expected_result",
+      "got_raw": "expected_result",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "expected_output",
+      "got": "expected_output",
+      "got_raw": "expected_output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "expected_output",
+      "got": "expected_output",
+      "got_raw": "expected_output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "expected_output",
+      "got": "expected_output",
+      "got_raw": "expected_output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "expected_output",
+      "got": "expected_output",
+      "got_raw": "expected_output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "expected_output",
+      "got": "expected_output",
+      "got_raw": "expected_output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[]",
+      "got": "\"foo3\"",
+      "got_raw": "\"foo3\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[docs]",
+      "got": "[docs]",
+      "got_raw": "[docs]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "{\"foo\": 3}",
+      "got": "{\"foo\": 3}",
+      "got_raw": "{\"foo\": 3}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "{\"foo\": 3}",
+      "got": "{\"foo\": 3}",
+      "got_raw": "{\"foo\": 3}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "{\"baz\": \"\"}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.18072040072196896,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "{\"baz\": \"\"}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.18072040072196896,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[compare_run]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.39640237166757364,
+      "edit_similarity": 0.26666666666666666
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[\"foo\", \"bar\"]",
+      "got": "[\"foo\", \"bar\"]",
+      "got_raw": "[\"foo\", \"bar\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[\"foo\", \"bar\"]",
+      "got": "\"foo bar\"",
+      "got_raw": "\"foo bar\"",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.782608695652174
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[compare_run] * 3",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.23784142300054423,
+      "edit_similarity": 0.21052631578947367
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[True, True, True]",
+      "got": "[True, True, True]",
+      "got_raw": "[True, True, True]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "{\"foo\": 3, \"bar\": 2}",
+      "got": "{\"foo\": 3, \"bar\": 2}",
+      "got_raw": "{\"foo\": 3, \"bar\": 2}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "{\"foo\": 3, \"bar\": 2}",
+      "got": "{\"foo\": 3, \"bar\": 2}",
+      "got_raw": "{\"foo\": 3, \"bar\": 2}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[_get_compare_run()]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.23784142300054423,
+      "edit_similarity": 0.18181818181818182
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[\"Human: bar\\nAI: foo\"]",
+      "got": "[\"foo\"]",
+      "got_raw": "[\"foo\"]",
+      "exact_match": false,
+      "code_bleu": 0.49999999999999994,
+      "edit_similarity": 0.4666666666666667
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[compare_run] * num_runs",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.23784142300054423,
+      "edit_similarity": 0.15384615384615385
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[\"Human: bar1\\nAI: foo1\"]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.39640237166757364,
+      "edit_similarity": 0.14814814814814814
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[EXAMPLES[2], EXAMPLES[0]]",
+      "got": "\"foo3\"",
+      "got_raw": "\"foo3\"",
+      "exact_match": false,
+      "code_bleu": 0.045454545454545456,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "{\"foo\": \"bar\", \"bar\": \"baz\"}",
+      "got": "\"bar\"",
+      "got_raw": "\"bar\"",
+      "exact_match": false,
+      "code_bleu": 0.0660670619445956,
+      "edit_similarity": 0.30303030303030304
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "{\"baz\": \"Human: bar\\nAI: foo\"}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.18072040072196896,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[Document(page_content=\"foo\")]",
+      "got": "[Document(page_content=\"foo\")]",
+      "got_raw": "[Document(page_content=\"foo\")]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[Document(page_content=\"foo\")]",
+      "got": "[Document(page_content=\"foo\")]",
+      "got_raw": "[Document(page_content=\"foo\")]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[Document(page_content=\"foo\")]",
+      "got": "[Document(page_content=\"foo\")]",
+      "got_raw": "[Document(page_content=\"foo\")]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[Document(page_content=\"foo\")]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.125
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[Document(page_content=\"foo\")]",
+      "got": "[Document(page_content=\"foo\")]",
+      "got_raw": "[Document(page_content=\"foo\")]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[Document(page_content=\"bar\")]",
+      "got": "[Document(page_content=\"bar\")]",
+      "got_raw": "[Document(page_content=\"bar\")]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[Document(page_content=\"baz\")]",
+      "got": "[Document(page_content=\"baz\")]",
+      "got_raw": "[Document(page_content=\"baz\")]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[Document(page_content=\"foo\")]",
+      "got": "[Document(page_content=\"foo\")]",
+      "got_raw": "[Document(page_content=\"foo\")]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[Document(page_content=\"foo\")]",
+      "got": "[Document(page_content=\"foo\")]",
+      "got_raw": "[Document(page_content=\"foo\")]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "{\"bar\": \"baz\", \"text1\": \"foo\"}",
+      "got": "\"Text1: This is a baz:\"",
+      "got_raw": "\"Text1: This is a baz:\"",
+      "exact_match": false,
+      "code_bleu": 0.05555555555555555,
+      "edit_similarity": 0.3018867924528302
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[Document(page_content=\"test\")]",
+      "got": "[Document(page_content=\"add\"",
+      "got_raw": "[Document(page_content=\"add\", metadata={\"page\": 1})]",
+      "exact_match": false,
+      "code_bleu": 0.6068300336776659,
+      "edit_similarity": 0.847457627118644
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[_get_compare_run()] * num_threads",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.16988673071467442,
+      "edit_similarity": 0.1111111111111111
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "{\"baz\": \"foo\\nHuman: bar1\\nAI: foo1\"}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.18072040072196896,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[EXAMPLES[2], EXAMPLES[0], EXAMPLES[1]]",
+      "got": "\"foo3\"",
+      "got_raw": "\"foo3\"",
+      "exact_match": false,
+      "code_bleu": 0.03125,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "{\"bar\": \"baz\", \"stop\": [\"foo\"], \"text1\": \"bar\"}",
+      "got": "{\"bar\": \"baz\"}",
+      "got_raw": "{\"bar\": \"baz\"}",
+      "exact_match": false,
+      "code_bleu": 0.26509024292235017,
+      "edit_similarity": 0.45901639344262296
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[Document(page_content=\"foo\", metadata={\"page\": 0})]",
+      "got": "[Document(page_content=\"foo\", metadata={\"page\": 0})]",
+      "got_raw": "[Document(page_content=\"foo\", metadata={\"page\": 0})]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[Document(page_content=\"foo\", metadata={\"page\": 0})]",
+      "got": "[Document(page_content=\"foo\", metadata={\"page\": 0})]",
+      "got_raw": "[Document(page_content=\"foo\", metadata={\"page\": 0})]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[Document(page_content=\"foo\", metadata={\"page\": 0})]",
+      "got": "[Document(page_content=\"foo\", metadata={\"page\": 0})]",
+      "got_raw": "[Document(page_content=\"foo\", metadata={\"page\": 0})]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[Document(page_content=\"foo\", metadata={\"page\": 0})]",
+      "got": "[Document(page_content=\"foo\", metadata={\"page\": 0})]",
+      "got_raw": "[Document(page_content=\"foo\", metadata={\"page\": 0})]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[Document(page_content=\"foo\", metadata={\"page\": 0})]",
+      "got": "[Document(page_content=\"foo\", metadata={\"page\": 0})]",
+      "got_raw": "[Document(page_content=\"foo\", metadata={\"page\": 0})]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "[Document(page_content=\"foo\", metadata={\"page\": \"0\"})]",
+      "got": "[Document(page_content=\"foo\", metadata={\"page\": \"0\"})]",
+      "got_raw": "[Document(page_content=\"foo\", metadata={\"page\": \"0\"})]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "docs[0]",
+      "got": "(\"foo\"",
+      "got_raw": "(\"foo\", {})",
+      "exact_match": false,
+      "code_bleu": 0.2259005009024612,
+      "edit_similarity": 0.15384615384615385
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "scores[1]",
+      "got": "scores[1]",
+      "got_raw": "scores[1]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "scores[1]",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "EXAMPLES[:1]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.23784142300054423,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "docs[0].metadata",
+      "got": "docs[0].metadata",
+      "got_raw": "docs[0].metadata",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "docs[0].page_content",
+      "got": "docs[0].page_content",
+      "got_raw": "docs[0].page_content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "EXAMPLES + [new_example]",
+      "got": "[short_question]",
+      "got_raw": "[short_question]",
+      "exact_match": false,
+      "code_bleu": 0.29999999999999993,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "expected_docstore.__dict__",
+      "got": "expected_docstore.__dict__",
+      "got_raw": "expected_docstore.__dict__",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "test_api_data[\"api_summary\"]",
+      "got": "test_api_data[\"api_response\"]",
+      "got_raw": "test_api_data[\"api_response\"]",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.7719298245614035
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "prompt_from_template.template",
+      "got": "prompt_from_template.template",
+      "got_raw": "prompt_from_template.template",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "prompt_from_template.input_variables",
+      "got": "prompt_from_template.input_variables",
+      "got_raw": "prompt_from_template.input_variables",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"2\"",
+      "got": "\"1",
+      "got_raw": "\"1 purple item remains on the desk.\"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"28\"",
+      "got": "\"12\"",
+      "got_raw": "\"12\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"2\\n\"",
+      "got": "\"2\"",
+      "got_raw": "\"2\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"NBA\"",
+      "got": "\"NBA\"",
+      "got_raw": "\"NBA\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"foo\"",
+      "got": "\"foo\"",
+      "got_raw": "\"foo\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"baz\"",
+      "got": "\"foo\"",
+      "got_raw": "\"foo\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"foo\"",
+      "got": "\"This",
+      "got_raw": "\"This is a baz:\"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"2\\n\"",
+      "got": "\"Hello",
+      "got_raw": "\"Hello World\"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.18181818181818182
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"bar\"",
+      "got": "\"foo\"",
+      "got_raw": "\"foo\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"bar\"",
+      "got": "\"bar\"",
+      "got_raw": "\"bar\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"foo\"",
+      "got": "\"foo\"",
+      "got_raw": "\"foo\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"foo\"",
+      "got": "\"foo\"",
+      "got_raw": "\"foo\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"1994\"",
+      "got": "\"1994\"",
+      "got_raw": "\"1994\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"2022\"",
+      "got": "\"2022\"",
+      "got_raw": "\"2022\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"foo \"",
+      "got": "\"foo \"",
+      "got_raw": "\"foo \"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"Albany\"",
+      "got": "\"New",
+      "got_raw": "\"New York City\"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"Albany\"",
+      "got": "\"New",
+      "got_raw": "\"New York City\"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"Albany\"",
+      "got": "\"New",
+      "got_raw": "\"New York\"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"Search\"",
+      "got": "\"Search\"",
+      "got_raw": "\"Search\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"search\"",
+      "got": "\"search\"",
+      "got_raw": "\"search\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"foo bar \"",
+      "got": "\"foo bar\"",
+      "got_raw": "\"foo bar\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9473684210526315
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"test_name\"",
+      "got": "\"test_name\"",
+      "got_raw": "\"test_name\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"Answer: 2\"",
+      "got": "\"Answer: 2\"",
+      "got_raw": "\"Answer: 2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"1994\\n1993\"",
+      "got": "\"1994\\n1993\"",
+      "got_raw": "\"1994\\n1993\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"search_api\"",
+      "got": "\"Search",
+      "got_raw": "\"Search API\"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.631578947368421
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"API result\"",
+      "got": "\"API result\"",
+      "got_raw": "\"API result\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"No Results\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"hello world\"",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"foo bar baz\"",
+      "got": "text",
+      "got_raw": "text",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"Bill Clinton\"",
+      "got": "\"David Chanoff has collaborated with Admiral Mike Mullen",
+      "got_raw": "\"David Chanoff has collaborated with Admiral Mike Mullen, who served as the ambassador to the United Kingdom under President George W. Bush.\"",
+      "exact_match": false,
+      "code_bleu": 0.11868405219520975,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"misalignment\"",
+      "got": "\"FooBarBaz\"",
+      "got_raw": "\"FooBarBaz\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.24
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"Final Answer\"",
+      "got": "\"Search\"",
+      "got_raw": "\"Search\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"No More Results\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.21052631578947367
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"El Palmar, Spain\"",
+      "got": "\"New York City\"",
+      "got_raw": "\"New York City\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"test_description\"",
+      "got": "\"test_description\"",
+      "got_raw": "\"test_description\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"This text is bad.\"",
+      "got": "text",
+      "got_raw": "text",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.34782608695652173
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"ID bar not found.\"",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "f\"Answer: {2**.5}\\n\"",
+      "got": "\"```python\\nprint(2**.5)\\n```\"",
+      "got_raw": "\"```python\\nprint(2**.5)\\n```\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.44
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"curses foiled again\"",
+      "got": "\"Action: BadAction\\nAction Input: misalignment\\n\"",
+      "got_raw": "\"Action: BadAction\\nAction Input: misalignment\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.22857142857142856
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"curses foiled again\"",
+      "got": "\"langchain was made in 2022\"",
+      "got_raw": "\"langchain was made in 2022\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.32653061224489793
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"I still don't know.\"",
+      "got": "\"I still don't know.\"",
+      "got_raw": "\"I still don't know.\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"file1.txt\\nfile2.txt\"",
+      "got": "\"file1.txt\\nfile2.txt\\n\"",
+      "got_raw": "\"file1.txt\\nfile2.txt\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9565217391304348
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"\\nFriend: bar\\nAI: foo\"",
+      "got": "{\"foo\": \"Friend: bar\\n\"}",
+      "got_raw": "{\"foo\": \"Friend: bar\\n\"}",
+      "exact_match": false,
+      "code_bleu": 0.2295748846661433,
+      "edit_similarity": 0.625
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"Barack Hussein Obama II\"",
+      "got": "\"Barack\"",
+      "got_raw": "\"Barack\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.48484848484848486
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"name 'foo' is not defined\"",
+      "got": "\"3\"",
+      "got_raw": "\"3\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.13333333333333333
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"\\nHuman: bar\\nAssistant: foo\"",
+      "got": "{\"foo\": \"foo\"}",
+      "got_raw": "{\"foo\": \"foo\"}",
+      "exact_match": false,
+      "code_bleu": 0.2295748846661433,
+      "edit_similarity": 0.22727272727272727
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "\"Question: {question}\\nAnswer:\"",
+      "got": "\"Question: {question}\\nAnswer:\"",
+      "got_raw": "\"Question: {question}\\nAnswer:\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "8",
+      "got": "8",
+      "got_raw": "8",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "1",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "7",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "0",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "1",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "3",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "1",
+      "got": "[0]",
+      "got_raw": "[0]",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "1",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "10",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "10",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "768",
+      "got": "768",
+      "got_raw": "768",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "768",
+      "got": "768",
+      "got_raw": "768",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "768",
+      "got": "768",
+      "got_raw": "768",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "len(texts)",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "len(text_input)",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "type(loaded_llm)",
+      "got": "type(loaded_llm)",
+      "got_raw": "type(loaded_llm)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "sorted(expected_output.split())",
+      "got": "sorted(expected_output.split())",
+      "got_raw": "sorted(expected_output.split())",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "sorted(\" \".join(expected_output.split()))",
+      "got": "sorted(\" \".join(expected_output.split()))",
+      "got_raw": "sorted(\" \".join(expected_output.split()))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "sorted(\" \".join(expected_output.split()))",
+      "got": "sorted(\" \".join(expected_output.split()))",
+      "got_raw": "sorted(\" \".join(expected_output.split()))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "subprocess.check_output(\"pwd\", shell=True).decode()",
+      "got": "Path.cwd()",
+      "got_raw": "Path.cwd()",
+      "exact_match": false,
+      "code_bleu": 0.13789192682003204,
+      "edit_similarity": 0.19672131147540983
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "microsoft/MM-REACT",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "storage.read()",
+      "got": "storage.read()",
+      "got_raw": "storage.read()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "[{'key': 'value'}]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.16988673071467442,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "6",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "0",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "-4",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "db.get(where('char') == 'a')",
+      "got": "db",
+      "got_raw": "db",
+      "exact_match": false,
+      "code_bleu": 0.049550296458446705,
+      "edit_similarity": 0.13333333333333333
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "d",
+      "got": "d",
+      "got_raw": "d",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "0",
+      "got": "q.foo",
+      "got_raw": "q.foo",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "''",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "\"Query()\"",
+      "got": "\"Query()\"",
+      "got_raw": "\"Query()\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "hash(where('asd'))",
+      "got": "hash(where('something').exists())",
+      "got_raw": "hash(where('something').exists())",
+      "exact_match": false,
+      "code_bleu": 0.3646285861936466,
+      "edit_similarity": 0.6274509803921569
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "hash(where('value'))",
+      "got": "hash(Query())",
+      "got_raw": "hash(Query())",
+      "exact_match": false,
+      "code_bleu": 0.32794537853815264,
+      "edit_similarity": 0.5454545454545454
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "where('value').exists()",
+      "got": "where('value')",
+      "got_raw": "where('value')",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7567567567567568
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "where('value')['val'].exists()",
+      "got": "where('value').has('val')",
+      "got_raw": "where('value').has('val')",
+      "exact_match": false,
+      "code_bleu": 0.37226793305847583,
+      "edit_similarity": 0.7272727272727273
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "\"QueryImpl('==', ('type',), 'peach')\"",
+      "got": "\"where(Fruit.type == 'peach')\"",
+      "got_raw": "\"where(Fruit.type == 'peach')\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5373134328358209
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "RuntimeError, match='Empty query was evaluated')",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.2711864406779661
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "3",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "item",
+      "got": "item",
+      "got_raw": "item",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "item2",
+      "got": "item2",
+      "got_raw": "item2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "[doc]",
+      "got": "[doc]",
+      "got_raw": "[doc]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "{'a': 1}",
+      "got": "{'a': 1}",
+      "got_raw": "{'a': 1}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "IOError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.631578947368421
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "exception)",
+      "got": "exception)",
+      "got_raw": "exception)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "TypeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "storage.read()",
+      "got": "storage.read()",
+      "got_raw": "storage.read()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "100",
+      "got": "100",
+      "got_raw": "100",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "[\"a\"]",
+      "got": "\"a\"",
+      "got_raw": "\"a\"",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "[\"b\"]",
+      "got": "\"b\"",
+      "got_raw": "\"b\"",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "cache",
+      "got": "cache",
+      "got_raw": "cache",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "TypeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "[\"c\", \"a\", \"d\"]",
+      "got": "['d', 'a', 'b']",
+      "got_raw": "['d', 'a', 'b']",
+      "exact_match": false,
+      "code_bleu": 0.23356898886410002,
+      "edit_similarity": 0.4666666666666667
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "[\"a\", \"b\", \"c\"]",
+      "got": "[\"c\", \"b\", \"a\"]",
+      "got_raw": "[\"c\", \"b\", \"a\"]",
+      "exact_match": false,
+      "code_bleu": 0.4111336169005197,
+      "edit_similarity": 0.8666666666666667
+    },
+    {
+      "repo": "msiemens/tinydb",
+      "expected": "[\"b\", \"c\", \"a\"]",
+      "got": "[\"b\", \"c\", \"a\"]",
+      "got_raw": "[\"b\", \"c\", \"a\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "css",
+      "got": "css",
+      "got_raw": "css",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "expected_css",
+      "got": "expected_css",
+      "got_raw": "expected_css",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "expected_content",
+      "got": "expected_content",
+      "got_raw": "expected_content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "output",
+      "got": "output",
+      "got_raw": "output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "{\"classes\": [\"compact\", \"display\", \"nowrap\"]}",
+      "got": "{\"classes\": [\"compact\", \"display\", \"nowrap\"]}",
+      "got_raw": "{\"classes\": [\"compact\", \"display\", \"nowrap\"]}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "ValueError, match=f\"Failed to load ITables config from {tmp_path / 'itables.toml'}\")",
+      "got": "pytest.raises(ValueError)",
+      "got_raw": " pytest.raises(ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.2095384381863861,
+      "edit_similarity": 0.2018348623853211
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "size_in_kb",
+      "got": "size_in_kb",
+      "got_raw": "size_in_kb",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "{1}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "set()",
+      "got": "{2",
+      "got_raw": "{2, 3}",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "result",
+      "got": "result",
+      "got_raw": "result",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "{2, 3}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.18072040072196896,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "dt_args",
+      "got": "dt_args",
+      "got_raw": "dt_args",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "0",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "256",
+      "got": "256",
+      "got_raw": "256",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "2**16",
+      "got": "2**16",
+      "got_raw": "2**16",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "2**20",
+      "got": "2**20",
+      "got_raw": "2**20",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "\"...\"",
+      "got": "df.iloc[0",
+      "got_raw": "df.iloc[0, 0]",
+      "exact_match": false,
+      "code_bleu": 0.2295748846661433,
+      "edit_similarity": 0.14285714285714285
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "(5, 5)",
+      "got": "(10, 10)",
+      "got_raw": "(10, 10)",
+      "exact_match": false,
+      "code_bleu": 0.32466791547509893,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "len(df)",
+      "got": "len(df)",
+      "got_raw": "len(df)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "len(dn)",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "(10, 5)",
+      "got": "(10, 20)",
+      "got_raw": "(10, 20)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "max_rows",
+      "got": "max_rows",
+      "got_raw": "max_rows",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "(10, 10)",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.11892071150027211,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "df.index)",
+      "got": "df.index)",
+      "got_raw": "df.index)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "df.columns",
+      "got": "df.columns",
+      "got_raw": "df.columns",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "{ \"classes\", \"style\", \"caption\", \"selected_rows\", }",
+      "got": "{\"format_floats_in_python\"}",
+      "got_raw": "{\"format_floats_in_python\"}",
+      "exact_match": false,
+      "code_bleu": 0.14999999999999997,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "{ \"table_html\", \"data_json\", \"column_filters\", \"layout\", \"order\", \"text_in_header_can_be_selected\", \"filtered_row_count\", \"downsampling_warning\", }",
+      "got": "set(df.columns)",
+      "got_raw": "set(df.columns)",
+      "exact_match": false,
+      "code_bleu": 0.061916859997111995,
+      "edit_similarity": 0.09876543209876543
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "allow_html",
+      "got": "allow_html",
+      "got_raw": "allow_html",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "html",
+      "got": "html",
+      "got_raw": "html",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "first_row",
+      "got": "first_row",
+      "got_raw": "first_row",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "len(df) <= 10",
+      "got": "True",
+      "got_raw": "True, {})",
+      "exact_match": false,
+      "code_bleu": 0.07142857142857142,
+      "edit_similarity": 0.11764705882352941
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "expected_tfoot",
+      "got": "expected_tfoot",
+      "got_raw": "expected_tfoot",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "len(df) <= min_rows",
+      "got": "len(df) <= min_rows)",
+      "got_raw": "len(df) <= min_rows)",
+      "exact_match": false,
+      "code_bleu": 0.8633400213704505,
+      "edit_similarity": 0.9743589743589743
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "\"line1\\nnew line2\\nline3\\n\"",
+      "got": "\"line1\\nnew line2\\nline3\\n\"",
+      "got_raw": "\"line1\\nnew line2\\nline3\\n\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "\"line1\\nnew line2\\nnew line2\\n\"",
+      "got": "\"new line2\\nline2\\nline2\\n\"",
+      "got_raw": "\"new line2\\nline2\\nline2\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7586206896551724
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "style.replace(\" \", \"\").removesuffix(\";\")",
+      "got": "style",
+      "got_raw": "style",
+      "exact_match": false,
+      "code_bleu": 0.04573873519241235,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "ValueError, match=\"cannot start with a number\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.3793103448275862
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "ValueError, match=\"was found 0 times in template\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.36065573770491804
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "ValueError, match=\"was found 2 times in template, expected 1.\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.2972972972972973
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "f\"https://www.unpkg.com/dt_for_itables@{bundle_version}/dt_bundle.js\"",
+      "got": "bundle_version",
+      "got_raw": "bundle_version",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.3373493975903614
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "f\"https://www.unpkg.com/dt_for_itables@{bundle_version}/dt_bundle.css\"",
+      "got": "UNPKG_DT_BUNDLE_CSS",
+      "got_raw": "UNPKG_DT_BUNDLE_CSS",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.06741573033707865
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "{ \"classes\": get_compact_classes(opt.classes), \"style\": get_compact_style(opt.style), \"data_json\": \"[[1], [2]]\", \"order\": [], \"display_logo_when_loading\": opt.display_logo_when_loading, \"warn_on_undocumented_option\": opt.warn_on_undocumented_option, \"warn_on_unexpected_option_type\": opt.warn_on_unexpected_option_type, \"dt_url\": opt.dt_url, \"layout\": {k: None for k in opt.layout}, \"text_in_header_can_be_selected\": True, }",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.01674939598595382,
+      "edit_similarity": 0.009389671361502348
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "{ \"classes\": get_compact_classes(opt.classes), \"style\": get_compact_style(opt.style), \"data_json\": '[[\"a\", 1], [\"b\", 2]]', \"order\": [], \"display_logo_when_loading\": opt.display_logo_when_loading, \"warn_on_undocumented_option\": opt.warn_on_undocumented_option, \"warn_on_unexpected_option_type\": opt.warn_on_unexpected_option_type, \"dt_url\": opt.dt_url, \"layout\": {k: None for k in opt.layout}, \"text_in_header_can_be_selected\": True, }",
+      "got": "{ \"data\": [ [1, 2] ], \"columns\": [ { \"data\": \"x\" }, { \"data\": \"index\" } ], \"style\": { \"class\": \"table-striped\" }, \"style_header\": { \"class\": \"table-dark\" }, \"style_cell\": { \"text_align\": \"center\" }, \"style_cell_conditional\": [ { \"if\": { \"column_id\": \"x\" }, \"text_align\": \"right\" } ] } }",
+      "got_raw": "{ \"data\": [ [1, 2] ], \"columns\": [ { \"data\": \"x\" }, { \"data\": \"index\" } ], \"style\": { \"class\": \"table-striped\" }, \"style_header\": { \"class\": \"table-dark\" }, \"style_cell\": { \"text_align\": \"center\" }, \"style_cell_conditional\": [ { \"if\": { \"column_id\": \"x\" }, \"text_align\": \"right\" } ] } }",
+      "exact_match": false,
+      "code_bleu": 0.06255817105022758,
+      "edit_similarity": 0.05
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "[[\"columnDefs\", 0, \"createdCell\"]]",
+      "got": "[\"_all\"]",
+      "got_raw": "[\"_all\"]",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666663,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "0",
+      "got": "MARIMO_APPS_PATH",
+      "got_raw": "MARIMO_APPS_PATH",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "dr_pd",
+      "got": "dr_pd",
+      "got_raw": "dr_pd",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "it_args",
+      "got": "it_args",
+      "got_raw": "it_args",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "it_args_nw",
+      "got": "it_args_nw",
+      "got_raw": "it_args_nw",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "dt_args",
+      "got": "dt_args",
+      "got_raw": "dt_args",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "table_html",
+      "got": "table_html",
+      "got_raw": "table_html",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "showIndex is True",
+      "got": "showIndex",
+      "got_raw": "showIndex",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.6923076923076923
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "dt_args[\"layout\"].values()",
+      "got": "dt_args[\"dom\"]",
+      "got_raw": "dt_args[\"dom\"]",
+      "exact_match": false,
+      "code_bleu": 0.25406637407730737,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "5",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "30",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "100",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "7e9",
+      "got": "100000",
+      "got_raw": "100000",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "500",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "rows",
+      "got": "rows",
+      "got_raw": "rows",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "cols",
+      "got": "cols",
+      "got_raw": "cols",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "5",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "30",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "100",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "7e9",
+      "got": "100",
+      "got_raw": "100",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "500",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "dt_args",
+      "got": "dt_args",
+      "got_raw": "dt_args",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "itable_args",
+      "got": "itable_args",
+      "got_raw": "itable_args",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "\"[[1, 4], [2, 5], [3, 6]]\"",
+      "got": "df.to_json()",
+      "got_raw": "df.to_json()",
+      "exact_match": false,
+      "code_bleu": 0.2295748846661433,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "[pl.Int64, pl.Float64, pl.String]",
+      "got": "pl.Series([\"int64\", \"float64\", \"str\"])",
+      "got_raw": "pl.Series([\"int64\", \"float64\", \"str\"])",
+      "exact_match": false,
+      "code_bleu": 0.15600574140522325,
+      "edit_similarity": 0.5633802816901409
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "[pl.Categorical, pl.Enum, pl.Int64]",
+      "got": "pl.Categorical",
+      "got_raw": "pl.Categorical",
+      "exact_match": false,
+      "code_bleu": 0.19405301890470336,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "'[[\"{\\\\\"C\\\\\",3}\"], [\"{\\\\\"A\\\\\",2}\"], [\"{\\\\\"B\\\\\",1}\"]]'",
+      "got": "\"X\"",
+      "got_raw": "\"X\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.07142857142857142
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "'[[\"a\", \"north\", 1], [\"b\", \"south\", 2], [\"a\", \"north\", 1], [\"c\", \"west\", 3]]'",
+      "got": "\"columns\"",
+      "got_raw": "\"columns\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.09302325581395349
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "\"[[1666767918216000000, 1699300000000, 0.9510565400123596, -0.30901700258255005]]\"",
+      "got": "[1666767918216000000, 1699300000000, 0.9510565400123596, -0.30901700258255005]",
+      "got_raw": "[1666767918216000000, 1699300000000, 0.9510565400123596, -0.30901700258255005]",
+      "exact_match": false,
+      "code_bleu": 0.10600313379512592,
+      "edit_similarity": 0.975
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "\"[[1666767918216000000, 1699300000000, \" '[\"0.951057\", 1], ' '[\"-0.309017\", 1]]]'",
+      "got": "[1666767918216000000, 1699300000000, 0.9510565400123596, -0.30901700258255005]",
+      "got_raw": "[1666767918216000000, 1699300000000, 0.9510565400123596, -0.30901700258255005]",
+      "exact_match": false,
+      "code_bleu": 0.10600313379512592,
+      "edit_similarity": 0.7044025157232704
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "'[[1, 0.1, \"x\"], ' \"[2, null, null], \" '[null, \"___NaN___\", \"z\"], ' '[4, 0.4, \"w\"]]'",
+      "got": "[1, 2, None, 4]",
+      "got_raw": "[1, 2, None, 4]",
+      "exact_match": false,
+      "code_bleu": 0.11868405219520975,
+      "edit_similarity": 0.24242424242424243
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "'[[1, [\"0.1\", 1], \"x\"], ' \"[2, [null, 0], null], \" '[null, [\"NaN\", 3], \"z\"], ' '[4, [\"0.4\", 2], \"w\"]]'",
+      "got": "\"[1, 0.1, \\\"x\\\"], [2, null, null], [null, \\\"___NaN___\\\", \\\"z\\\"], [4, 0.4, \\\"w\\\"]\"",
+      "got_raw": "\"[1, 0.1, \\\"x\\\"], [2, null, null], [null, \\\"___NaN___\\\", \\\"z\\\"], [4, 0.4, \\\"w\\\"]\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.7103825136612022
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "0",
+      "got": "SHINY_APPS_PY_FILES",
+      "got_raw": "SHINY_APPS_PY_FILES",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "IndexError, match=\"Selected rows out of range\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.2413793103448276
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "captured.out",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "0",
+      "got": "STREAMLIT_APPS_PY_FILES",
+      "got_raw": "STREAMLIT_APPS_PY_FILES",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "df",
+      "got": "df",
+      "got_raw": "df",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "opt.style",
+      "got": "opt.style",
+      "got_raw": "opt.style",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "mwouts/itables",
+      "expected": "opt.classes",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "3",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "4",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "8",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "0",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "3",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "5",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "3",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "33 - 7",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "33 - 5",
+      "got": "33 - 5",
+      "got_raw": "33 - 5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "123 * 2",
+      "got": "123",
+      "got_raw": "123",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "proxy.url",
+      "got": "proxy.url",
+      "got_raw": "proxy.url",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "cause.new",
+      "got": "cause.new",
+      "got_raw": "cause.new",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "cause.old",
+      "got": "cause.old",
+      "got_raw": "cause.old",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "cause.memo",
+      "got": "cause.memo",
+      "got_raw": "cause.memo",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "cause.body",
+      "got": "cause.body",
+      "got_raw": "cause.body",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "cause.diff",
+      "got": "cause.diff",
+      "got_raw": "cause.diff",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "cause.type",
+      "got": "cause.type",
+      "got_raw": "cause.type",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "view2._src",
+      "got": "data1",
+      "got_raw": "data1",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "certpkey[0]",
+      "got": "certpkey",
+      "got_raw": "certpkey",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.8421052631578947
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "Reason.GONE",
+      "got": "Reason.GONE",
+      "got_raw": "Reason.GONE",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "Reason.FREE",
+      "got": "Reason.NO_FINALIZER",
+      "got_raw": "Reason.NO_FINALIZER",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "Reason.NOOP",
+      "got": "Reason.NO_OP",
+      "got_raw": "Reason.NO_OP",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.9565217391304348
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "body['spec']",
+      "got": "body['spec']",
+      "got_raw": "body['spec']",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "d",
+      "got": "d",
+      "got_raw": "d",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "d",
+      "got": "d",
+      "got_raw": "d",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "fn",
+      "got": "fn",
+      "got_raw": "fn",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "id",
+      "got": "id",
+      "got_raw": "id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "d2",
+      "got": "d2",
+      "got_raw": "d2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "fn",
+      "got": "fn",
+      "got_raw": "fn",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "fn",
+      "got": "fn",
+      "got_raw": "fn",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "url",
+      "got": "url",
+      "got_raw": "url",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "exc",
+      "got": "exc",
+      "got_raw": "exc",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "old",
+      "got": "old",
+      "got_raw": "old",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "new",
+      "got": "new",
+      "got_raw": "new",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "old",
+      "got": "old",
+      "got_raw": "old",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "new",
+      "got": "new",
+      "got_raw": "new",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "obj",
+      "got": "obj",
+      "got_raw": "obj",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "obj",
+      "got": "obj",
+      "got_raw": "obj",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "ts0",
+      "got": "tsA_triggered",
+      "got_raw": "tsA_triggered",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "obj",
+      "got": "obj",
+      "got_raw": "obj",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "obj",
+      "got": "obj",
+      "got_raw": "obj",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "uid",
+      "got": "uid",
+      "got_raw": "uid",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "code",
+      "got": "code",
+      "got_raw": "code",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "info",
+      "got": "info",
+      "got_raw": "info",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "key1",
+      "got": "key1",
+      "got_raw": "key1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "key2",
+      "got": "key1",
+      "got_raw": "key1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "obj1",
+      "got": "obj1",
+      "got_raw": "obj1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "memo",
+      "got": "memo",
+      "got_raw": "memo",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "body",
+      "got": "body",
+      "got_raw": "body",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "type",
+      "got": "type",
+      "got_raw": "type",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "diff",
+      "got": "diff",
+      "got_raw": "diff",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "when",
+      "got": "when",
+      "got_raw": "when",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "obj1",
+      "got": "obj1",
+      "got_raw": "obj1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "obj2",
+      "got": "obj2",
+      "got_raw": "obj2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "DIFF",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "obj1",
+      "got": "obj1",
+      "got_raw": "obj1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "obj2",
+      "got": "obj2",
+      "got_raw": "obj2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "keys",
+      "got": "keys",
+      "got_raw": "keys",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "body",
+      "got": "body",
+      "got_raw": "body",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "REF1",
+      "got": "REF1",
+      "got_raw": "REF1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "REF1",
+      "got": "REF1",
+      "got_raw": "REF1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "REF2",
+      "got": "REF2",
+      "got_raw": "REF2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "cert1",
+      "got": "cert1",
+      "got_raw": "cert1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "cert2",
+      "got": "cert2",
+      "got_raw": "cert2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "True",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "()",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "()",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{1}",
+      "got": "{1}",
+      "got_raw": "{1}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{r1}",
+      "got": "r1",
+      "got_raw": "r1",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{r1}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{r2}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{''}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "[5.0]",
+      "got": "[0.0",
+      "got_raw": "[0.0, 0.0, 0.0, 0.0, 0.0]",
+      "exact_match": false,
+      "code_bleu": 0.35818997727451396,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{123}",
+      "got": "{123}",
+      "got_raw": "{123}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{123}",
+      "got": "{memory}",
+      "got_raw": "{memory}",
+      "exact_match": false,
+      "code_bleu": 0.49999999999999994,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{123}",
+      "got": "{123}",
+      "got_raw": "{123}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{456}",
+      "got": "{456}",
+      "got_raw": "{456}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "[123]",
+      "got": "[123]",
+      "got_raw": "[123]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{None}",
+      "got": "{key}",
+      "got_raw": "{key}",
+      "exact_match": false,
+      "code_bleu": 0.49999999999999994,
+      "edit_similarity": 0.5454545454545454
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{None}",
+      "got": "{123}",
+      "got_raw": "{123}",
+      "exact_match": false,
+      "code_bleu": 0.49999999999999994,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{None}",
+      "got": "{123}",
+      "got_raw": "{123}",
+      "exact_match": false,
+      "code_bleu": 0.49999999999999994,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{task}",
+      "got": "[task]",
+      "got_raw": "[task]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{'xyz'}",
+      "got": "{'xyz'}",
+      "got_raw": "{'xyz'}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{'ns1'}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{'ns1'}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{r1ns1}",
+      "got": "{r1ns1_task",
+      "got_raw": "{r1ns1_task, r1ns2_task, r2ns1_task, r2ns2_task}",
+      "exact_match": false,
+      "code_bleu": 0.35818997727451396,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{peer1}",
+      "got": "{peer1}",
+      "got_raw": "{peer1}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{peerN}",
+      "got": "{peerN}",
+      "got_raw": "{peerN}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{'id1'}",
+      "got": "{'alive'}",
+      "got_raw": "{'alive'}",
+      "exact_match": false,
+      "code_bleu": 0.49999999999999994,
+      "edit_similarity": 0.625
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "[{}, {}]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.16988673071467442,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "{r1, r2}",
+      "got": "r2",
+      "got_raw": "r2",
+      "exact_match": false,
+      "code_bleu": 0.11892071150027211,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "['oops!']",
+      "got": "[\"oops!\"]",
+      "got_raw": "[\"oops!\"]",
+      "exact_match": false,
+      "code_bleu": 0.49999999999999994,
+      "edit_similarity": 0.7777777777777778
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "['field']",
+      "got": "['field']",
+      "got_raw": "['field']",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "['value']",
+      "got": "['value']",
+      "got_raw": "['value']",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "\"\"",
+      "got": "\"456\"",
+      "got_raw": "\"456\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "''",
+      "got": "'kind1'",
+      "got_raw": "'kind1'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "''",
+      "got": "'v1'",
+      "got_raw": "'v1'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'a'",
+      "got": "'a'",
+      "got_raw": "'a'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'b'",
+      "got": "'b'",
+      "got_raw": "'b'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'a'",
+      "got": "'a'",
+      "got_raw": "'a'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'b'",
+      "got": "'b'",
+      "got_raw": "'b'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'c'",
+      "got": "'c'",
+      "got_raw": "'c'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'d'",
+      "got": "'d'",
+      "got_raw": "'d'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'a'",
+      "got": "'a'",
+      "got_raw": "'a'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'b'",
+      "got": "'b'",
+      "got_raw": "'b'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'a'",
+      "got": "'a'",
+      "got_raw": "'a'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'b'",
+      "got": "'b'",
+      "got_raw": "'b'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'x'",
+      "got": "'x'",
+      "got_raw": "'x'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'1'",
+      "got": "'3'",
+      "got_raw": "'3'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'2'",
+      "got": "'2'",
+      "got_raw": "'2'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'a'",
+      "got": "'a'",
+      "got_raw": "'a'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'xx'",
+      "got": "'xx'",
+      "got_raw": "'xx'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'ns'",
+      "got": "'default'",
+      "got_raw": "'default'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'ns'",
+      "got": "'ns'",
+      "got_raw": "'ns'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'id'",
+      "got": "'id'",
+      "got_raw": "'id'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'\\n'",
+      "got": "''",
+      "got_raw": "''",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'xyz'",
+      "got": "'xyz'",
+      "got_raw": "'xyz'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'xyz'",
+      "got": "'xyz'",
+      "got_raw": "'xyz'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'xyz'",
+      "got": "'xyz'",
+      "got_raw": "'xyz'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'xyz'",
+      "got": "'xyz'",
+      "got_raw": "'xyz'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "\"boo\"",
+      "got": "\"boo\"",
+      "got_raw": "\"boo\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "'tkn'",
+      "got": "'tkn'",
+      "got_raw": "'tkn'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "set()",
+      "got": "{'ADDED'",
+      "got_raw": "{'ADDED', 'MODIFIED'}",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "set()",
+      "got": "{123}",
+      "got_raw": "{123}",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "set()",
+      "got": "{456}",
+      "got_raw": "{456}",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "set()",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "set()",
+      "got": "set()",
+      "got_raw": "set()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "nolar/kopf",
+      "expected": "hash(d2)",
+      "got": "hash(d2)",
+      "got_raw": "hash(d2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "d)",
+      "got": "d)",
+      "got_raw": "d)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "n)",
+      "got": "n)",
+      "got_raw": "n)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a)",
+      "got": "a)",
+      "got_raw": "a)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b)",
+      "got": "b)",
+      "got_raw": "b)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "c)",
+      "got": "c)",
+      "got_raw": "c)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a)",
+      "got": "a)",
+      "got_raw": "a)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a)",
+      "got": "a)",
+      "got_raw": "a)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b)",
+      "got": "b)",
+      "got_raw": "b)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "c)",
+      "got": "c)",
+      "got_raw": "c)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "d)",
+      "got": "d)",
+      "got_raw": "d)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a)",
+      "got": "a)",
+      "got_raw": "a)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a)",
+      "got": "a)",
+      "got_raw": "a)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a)",
+      "got": "b)",
+      "got_raw": "b)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b)",
+      "got": "b)",
+      "got_raw": "b)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a)",
+      "got": "a)",
+      "got_raw": "a)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b)",
+      "got": "b)",
+      "got_raw": "b)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "c)",
+      "got": "c)",
+      "got_raw": "c)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a)",
+      "got": "a)",
+      "got_raw": "a)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b)",
+      "got": "b)",
+      "got_raw": "b)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "c)",
+      "got": "c)",
+      "got_raw": "c)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a)",
+      "got": "a)",
+      "got_raw": "a)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b)",
+      "got": "b)",
+      "got_raw": "b)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "c)",
+      "got": "c)",
+      "got_raw": "c)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a)",
+      "got": "a)",
+      "got_raw": "a)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a)",
+      "got": "a)",
+      "got_raw": "a)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b)",
+      "got": "b)",
+      "got_raw": "b)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "c)",
+      "got": "c)",
+      "got_raw": "c)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "d)",
+      "got": "d)",
+      "got_raw": "d)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "e)",
+      "got": "e)",
+      "got_raw": "e)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a)",
+      "got": "a)",
+      "got_raw": "a)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b)",
+      "got": "b)",
+      "got_raw": "b)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "c)",
+      "got": "c)",
+      "got_raw": "c)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "d)",
+      "got": "d)",
+      "got_raw": "d)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b)",
+      "got": "b)",
+      "got_raw": "b)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a)",
+      "got": "a)",
+      "got_raw": "a)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "c)",
+      "got": "c)",
+      "got_raw": "c)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "d)",
+      "got": "d)",
+      "got_raw": "d)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "e)",
+      "got": "e)",
+      "got_raw": "e)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a)",
+      "got": "a)",
+      "got_raw": "a)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a)",
+      "got": "a)",
+      "got_raw": "a)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b)",
+      "got": "b)",
+      "got_raw": "b)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "c)",
+      "got": "c)",
+      "got_raw": "c)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "e)",
+      "got": "e)",
+      "got_raw": "e)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a)",
+      "got": "a)",
+      "got_raw": "a)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "v)",
+      "got": "v)",
+      "got_raw": "v)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b)",
+      "got": "b)",
+      "got_raw": "b)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "sh)",
+      "got": "sh)",
+      "got_raw": "sh)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "tc)",
+      "got": "len(tc.records))",
+      "got_raw": "len(tc.records))",
+      "exact_match": false,
+      "code_bleu": 0.20556680845025982,
+      "edit_similarity": 0.3157894736842105
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "one)",
+      "got": "same)",
+      "got_raw": "same)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got)",
+      "got": "got)",
+      "got_raw": "got)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got)",
+      "got": "got)",
+      "got_raw": "got)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "out)",
+      "got": "out)",
+      "got_raw": "out)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "ttl)",
+      "got": "ttl)",
+      "got_raw": "ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got)",
+      "got": "got)",
+      "got_raw": "got)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got)",
+      "got": "got)",
+      "got_raw": "got)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got)",
+      "got": "got)",
+      "got_raw": "got)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "one)",
+      "got": "one)",
+      "got_raw": "one)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "two)",
+      "got": "two)",
+      "got_raw": "two)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "msg)",
+      "got": "msg)",
+      "got_raw": "msg)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got)",
+      "got": "got)",
+      "got_raw": "got)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got)",
+      "got": "got)",
+      "got_raw": "got)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got)",
+      "got": "got)",
+      "got_raw": "got)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got)",
+      "got": "got)",
+      "got_raw": "got)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got)",
+      "got": "got)",
+      "got_raw": "got)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got)",
+      "got": "got)",
+      "got_raw": "got)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got)",
+      "got": "got)",
+      "got_raw": "got)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got)",
+      "got": "got)",
+      "got_raw": "got)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got)",
+      "got": "got)",
+      "got_raw": "got)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got)",
+      "got": "value)",
+      "got_raw": "value)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got)",
+      "got": "got)",
+      "got_raw": "got)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got)",
+      "got": "got)",
+      "got_raw": "got)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got)",
+      "got": "got)",
+      "got_raw": "got)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "str)",
+      "got": "str)",
+      "got_raw": "str)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "int)",
+      "got": "int)",
+      "got_raw": "int)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "same)",
+      "got": "same)",
+      "got_raw": "same)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "data)",
+      "got": "data)",
+      "got_raw": "data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "list)",
+      "got": "list)",
+      "got_raw": "list)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "plan)",
+      "got": "plan)",
+      "got_raw": "plan)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "plan)",
+      "got": "plan)",
+      "got_raw": "plan)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "plan)",
+      "got": "plan)",
+      "got_raw": "plan)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "data)",
+      "got": "['include']",
+      "got_raw": "['include']",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.125
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "value)",
+      "got": "value)",
+      "got_raw": "value)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "found)",
+      "got": "found)",
+      "got_raw": "found)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "other)",
+      "got": "other)",
+      "got_raw": "other)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "len(d))",
+      "got": "len(d))",
+      "got_raw": "len(d))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "set([a]))",
+      "got": "[a])",
+      "got_raw": "[a])",
+      "exact_match": false,
+      "code_bleu": 0.5714285714285714,
+      "edit_similarity": 0.6153846153846154
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "len(data))",
+      "got": "len(data))",
+      "got_raw": "len(data))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "len(values))",
+      "got": "len(values))",
+      "got_raw": "len(values))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "set([a, b]))",
+      "got": "set([a, b]))",
+      "got_raw": "set([a, b]))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "set([a, c]))",
+      "got": "[a, c])",
+      "got_raw": "[a, c])",
+      "exact_match": false,
+      "code_bleu": 0.6666666666666666,
+      "edit_similarity": 0.7368421052631579
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "len(changes))",
+      "got": "len(changes))",
+      "got_raw": "len(changes))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "buf.getvalue())",
+      "got": "'*.1.1:",
+      "got_raw": "'*.1.1: 42\\n*.2.1: 44\\n*.11.1: 43\\n')",
+      "exact_match": false,
+      "code_bleu": 0.22751007189242223,
+      "edit_similarity": 0.09090909090909091
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "tuple(d.keys()))",
+      "got": "[self.plain]",
+      "got_raw": "[self.plain]",
+      "exact_match": false,
+      "code_bleu": 0.15167338126161484,
+      "edit_similarity": 0.14285714285714285
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "len(txt.values))",
+      "got": "len(txt.values))",
+      "got_raw": "len(txt.values))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "sorted(changes))",
+      "got": "changes)",
+      "got_raw": "changes)",
+      "exact_match": false,
+      "code_bleu": 0.28284271247461906,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "idna_decode(got))",
+      "got": "idna_decode(got))",
+      "got_raw": "idna_decode(got))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "tuple(d.items()))",
+      "got": "[(self.almost",
+      "got_raw": "[(self.almost, 43)]",
+      "exact_match": false,
+      "code_bleu": 0.16785418874204266,
+      "edit_similarity": 0.13333333333333333
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "len(got.records))",
+      "got": "len(got.records))",
+      "got_raw": "len(got.records))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "len(zone.records))",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": false,
+      "code_bleu": 0.153509990260506,
+      "edit_similarity": 0.1
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "len(zone.records))",
+      "got": "len(zone.records))",
+      "got_raw": "len(zone.records))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "len(plan.changes))",
+      "got": "len(plan.changes))",
+      "got_raw": "len(plan.changes))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "len(zone.records))",
+      "got": "len(zone.records))",
+      "got_raw": "len(zone.records))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "str(ctx.exception))",
+      "got": "'Invalid",
+      "got_raw": "'Invalid IDNA domain')",
+      "exact_match": false,
+      "code_bleu": 0.1290860005156921,
+      "edit_similarity": 0.07407407407407407
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "str(ctx.exception))",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": false,
+      "code_bleu": 0.153509990260506,
+      "edit_similarity": 0.09523809523809523
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "str(ctx.exception))",
+      "got": "\"bad\"",
+      "got_raw": "\"bad\"",
+      "exact_match": false,
+      "code_bleu": 0.07142857142857142,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "value.template({}))",
+      "got": "value)",
+      "got_raw": "value)",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.48
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "len(single.values))",
+      "got": "len(single.values))",
+      "got_raw": "len(single.values))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "str(ctx.exception))",
+      "got": "str(ctx.exception))",
+      "got_raw": "str(ctx.exception))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "str(ctx.exception))",
+      "got": "('include-doesnt-exist.yaml",
+      "got_raw": "('include-doesnt-exist.yaml not found')",
+      "exact_match": false,
+      "code_bleu": 0.15619699684601276,
+      "edit_similarity": 0.2608695652173913
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "GeoCodes.parse('NA'))",
+      "got": ".zone.add_record(Record.new(self.zone",
+      "got_raw": ".zone.add_record(Record.new(self.zone, 'unit.tests.', 'A', '1.2.3.4'))",
+      "exact_match": false,
+      "code_bleu": 0.11498759556447223,
+      "edit_similarity": 0.1724137931034483
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.changes(a, target))",
+      "got": "a.changes(target, a))",
+      "got_raw": "a.changes(target, a))",
+      "exact_match": false,
+      "code_bleu": 0.49393827371153703,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.changes(b, target))",
+      "got": "b.changes(a, target))",
+      "got_raw": "b.changes(a, target))",
+      "exact_match": false,
+      "code_bleu": 0.5555238068023581,
+      "edit_similarity": 0.9047619047619048
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.changes(c, target))",
+      "got": "c.changes(a, target))",
+      "got_raw": "c.changes(a, target))",
+      "exact_match": false,
+      "code_bleu": 0.5555238068023581,
+      "edit_similarity": 0.9047619047619048
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "len(filtered.records))",
+      "got": "len(filtered.records))",
+      "got_raw": "len(filtered.records))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "len(with_dot.records))",
+      "got": "len(with_dot.records))",
+      "got_raw": "len(with_dot.records))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "str(context.exception))",
+      "got": "True)",
+      "got_raw": "True)",
+      "exact_match": false,
+      "code_bleu": 0.153509990260506,
+      "edit_similarity": 0.21428571428571427
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "tuple(d.decoded_keys()))",
+      "got": "self.utf8)",
+      "got_raw": "self.utf8)",
+      "exact_match": false,
+      "code_bleu": 0.17672682861490013,
+      "edit_similarity": 0.11764705882352941
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "_find(got, 'has').value)",
+      "got": "missing.value)",
+      "got_raw": "missing.value)",
+      "exact_match": false,
+      "code_bleu": 0.2972179133228542,
+      "edit_similarity": 0.5263157894736842
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "_find(got, 'sub').value)",
+      "got": "got)",
+      "got_raw": "got)",
+      "exact_match": false,
+      "code_bleu": 0.1321341238891912,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a < b)",
+      "got": "a == b)",
+      "got_raw": "a == b)",
+      "exact_match": false,
+      "code_bleu": 0.3860973950960897,
+      "edit_similarity": 0.7692307692307693
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b.ttl)",
+      "got": "b.ttl)",
+      "got_raw": "b.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.ttl)",
+      "got": "a.ttl)",
+      "got_raw": "a.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b.data)",
+      "got": "b.data)",
+      "got_raw": "b.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "c.data)",
+      "got": "c.data)",
+      "got_raw": "c.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b.name)",
+      "got": "b.name)",
+      "got_raw": "b.name)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "c.name)",
+      "got": "c.name)",
+      "got_raw": "c.name)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.name)",
+      "got": "a.name)",
+      "got_raw": "a.name)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.fqdn)",
+      "got": "a.fqdn)",
+      "got_raw": "a.fqdn)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.data)",
+      "got": "a.data)",
+      "got_raw": "a.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b.data)",
+      "got": "b.data)",
+      "got_raw": "b.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.name)",
+      "got": "a.name)",
+      "got_raw": "a.name)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.fqdn)",
+      "got": "a.fqdn)",
+      "got_raw": "a.fqdn)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.data)",
+      "got": "a.data)",
+      "got_raw": "a.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b.data)",
+      "got": "b.data)",
+      "got_raw": "b.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.name)",
+      "got": "a.name)",
+      "got_raw": "a.name)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.fqdn)",
+      "got": "a.zone.name)",
+      "got_raw": "a.zone.name)",
+      "exact_match": false,
+      "code_bleu": 0.31239399369202553,
+      "edit_similarity": 0.42105263157894735
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.name)",
+      "got": "a.name)",
+      "got_raw": "a.name)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.fqdn)",
+      "got": "a.fqdn)",
+      "got_raw": "a.fqdn)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.name)",
+      "got": "a.name)",
+      "got_raw": "a.name)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.fqdn)",
+      "got": "a.fqdn)",
+      "got_raw": "a.fqdn)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.name)",
+      "got": "a.name)",
+      "got_raw": "a.name)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.fqdn)",
+      "got": "a.fqdn)",
+      "got_raw": "a.fqdn)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.data)",
+      "got": "a.data)",
+      "got_raw": "a.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b.data)",
+      "got": "b.data)",
+      "got_raw": "b.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.name)",
+      "got": "a.name)",
+      "got_raw": "a.name)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.fqdn)",
+      "got": "a.fqdn)",
+      "got_raw": "a.fqdn)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.data)",
+      "got": "a.data)",
+      "got_raw": "a.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b.data)",
+      "got": "b.data)",
+      "got_raw": "b.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.name)",
+      "got": "a.name)",
+      "got_raw": "a.name)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.fqdn)",
+      "got": "a.fqdn)",
+      "got_raw": "a.fqdn)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.data)",
+      "got": "a.data)",
+      "got_raw": "a.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b.data)",
+      "got": "b.data)",
+      "got_raw": "b.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.name)",
+      "got": "a.name)",
+      "got_raw": "a.name)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.fqdn)",
+      "got": "a.fqdn)",
+      "got_raw": "a.fqdn)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a.data)",
+      "got": "a.data)",
+      "got_raw": "a.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "b.data)",
+      "got": "b.data)",
+      "got_raw": "b.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "ptr.ttl)",
+      "got": "ptr.ttl)",
+      "got_raw": "ptr.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "a._type)",
+      "got": "a.type)",
+      "got_raw": "a.type)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.9333333333333333
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "ptr.fqdn)",
+      "got": "ptr.name)",
+      "got_raw": "ptr.name)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "geo.data)",
+      "got": "geo.data)",
+      "got_raw": "geo.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got.path)",
+      "got": "got['path'])",
+      "got_raw": "got['path'])",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.7619047619047619
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "ptr.value)",
+      "got": "ptr.value)",
+      "got_raw": "ptr.value)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "ptr_2.ttl)",
+      "got": "ptr_1.ttl)",
+      "got_raw": "ptr_1.ttl)",
+      "exact_match": false,
+      "code_bleu": 0.668740304976422,
+      "edit_similarity": 0.9
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "proc.uuid)",
+      "got": "proc.should_update(self.meta_needs_update))",
+      "got_raw": "proc.should_update(self.meta_needs_update))",
+      "exact_match": false,
+      "code_bleu": 0.19960198807747329,
+      "edit_similarity": 0.3018867924528302
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "proc.time)",
+      "got": "proc.value)",
+      "got_raw": "proc.value)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "meta.name)",
+      "got": "meta.name)",
+      "got_raw": "meta.name)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got.value)",
+      "got": "got.value)",
+      "got_raw": "got.value)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "zone.name)",
+      "got": "zone.name)",
+      "got_raw": "zone.name)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "ptr_1.fqdn)",
+      "got": "ptr_1.name)",
+      "got_raw": "ptr_1.name)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.7272727272727273
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "ptr_2.fqdn)",
+      "got": "ptr_2.fqdn)",
+      "got_raw": "ptr_2.fqdn)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "ptr.values)",
+      "got": "ptr.value)",
+      "got_raw": "ptr.value)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "meta._type)",
+      "got": "meta.type)",
+      "got_raw": "meta.type)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "record.ttl)",
+      "got": "record.ttl)",
+      "got_raw": "record.ttl)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "noop.value)",
+      "got": "noop.value)",
+      "got_raw": "noop.value)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got.values)",
+      "got": "got.values)",
+      "got_raw": "got.values)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "got.digest)",
+      "got": "got['digest'])",
+      "got_raw": "got['digest'])",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "other.data)",
+      "got": "other.data)",
+      "got_raw": "other.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "2)",
+      "got": "2)",
+      "got_raw": "2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "'TXT')",
+      "got": "None)",
+      "got_raw": "None)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.18181818181818182
+    },
+    {
+      "repo": "octodns/octodns",
+      "expected": "'testvalue')",
+      "got": "'testvalue')",
+      "got_raw": "'testvalue')",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "b'\\x00\\x00'",
+      "got": "b\"\"",
+      "got_raw": "b\"\"",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.14285714285714285
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "BINARY_ADDRESS",
+      "got": "b58decode_check(BITCOIN_ADDRESS)",
+      "got_raw": "b58decode_check(BITCOIN_ADDRESS)",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.4782608695652174
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "BITCOIN_ADDRESS",
+      "got": "BITCOIN_ADDRESS",
+      "got_raw": "BITCOIN_ADDRESS",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "BITCOIN_ADDRESS[:1]",
+      "got": "\"B1\"",
+      "got_raw": "\"B1\"",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.17391304347826086
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "MAIN_PUBKEY_HASH + PUBKEY_HASH",
+      "got": "BITCOIN_ADDRESS",
+      "got_raw": "BITCOIN_ADDRESS",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.26666666666666666
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "0",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "point",
+      "got": "point",
+      "got_raw": "point",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "Y_CORRECT",
+      "got": "Y_CORRECT",
+      "got_raw": "Y_CORRECT",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "Y_INCORRECT",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.13333333333333333
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "'Point(x=5, y=10)'",
+      "got": "\"Point(x=5, y=10)\"",
+      "got_raw": "\"Point(x=5, y=10)\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "'main'",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "'test'",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "PUBKEY_HASH",
+      "got": "PUBKEY_HASH",
+      "got_raw": "PUBKEY_HASH",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "PAY2SH_HASH",
+      "got": "PAY2SH_HASH",
+      "got_raw": "PAY2SH_HASH",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "BITCOIN_ADDRESS",
+      "got": "BITCOIN_ADDRESS",
+      "got_raw": "BITCOIN_ADDRESS",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "addr",
+      "got": "addr",
+      "got_raw": "addr",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "hex_to_bytes(hexscript)",
+      "got": "hex_to_bytes(hexscript)",
+      "got_raw": "hex_to_bytes(hexscript)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "3",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "b''",
+      "got": "b'\\xff\\xff\\xff\\xff'",
+      "got_raw": "b'\\xff\\xff\\xff\\xff'",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.2727272727272727
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "HEX",
+      "got": "bytes_to_hex(BYTES_BIG)",
+      "got_raw": "bytes_to_hex(BYTES_BIG)",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.07692307692307693
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "BIG_INT",
+      "got": "BIG_INT",
+      "got_raw": "BIG_INT",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "b'\\x00'",
+      "got": "b''",
+      "got_raw": "b''",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "BYTES_BIG",
+      "got": "BYTES_BIG",
+      "got_raw": "BYTES_BIG",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "HEX.upper()",
+      "got": "'544838e2aa4ed75e61583793e7c6a30285'",
+      "got_raw": "'544838e2aa4ed75e61583793e7c6a30285'",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.0425531914893617
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "BYTES_LITTLE",
+      "got": "BYTES_LITTLE",
+      "got_raw": "BYTES_LITTLE",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "ODD_HEX_BYTES",
+      "got": "ODD_HEX_BYTES",
+      "got_raw": "ODD_HEX_BYTES",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "Decimal('0.8')",
+      "got": "Decimal('0.8')",
+      "got_raw": "Decimal('0.8')",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "[ '4f', 'ad', 'd1', '97', '73', '28', 'c1', '1e', 'fc', '1c', '1d', '8a', '78', '1a', 'a6', 'b9', '67', '79', '84', 'd3', 'e0', 'bd', '0b', 'fc', '52', 'b9', 'f3', 'b0', '38', '85', 'a0', '0', ]",
+      "got": "[ODD_HEX[:2]]",
+      "got_raw": "[ODD_HEX[:2]]",
+      "exact_match": false,
+      "code_bleu": 0.021802540290179074,
+      "edit_similarity": 0.028985507246376812
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "pk",
+      "got": "pk",
+      "got_raw": "pk",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "262",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "188",
+      "got": "262",
+      "got_raw": "262",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "113",
+      "got": "131.5",
+      "got_raw": "131.5",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "298",
+      "got": "188",
+      "got_raw": "188",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "140",
+      "got": "166.5",
+      "got_raw": "166.5",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "None",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "initial",
+      "got": "initial",
+      "got_raw": "initial",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "PrivateKey",
+      "got": "PrivateKey",
+      "got_raw": "PrivateKey",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "TypeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "30",
+      "got": "30",
+      "got_raw": "30",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "updated",
+      "got": "updated",
+      "got_raw": "updated",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "cached_time",
+      "got": "cached_time",
+      "got_raw": "cached_time",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "get_fee(fast=False)",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "7",
+      "got": "7",
+      "got_raw": "7",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "91",
+      "got": "104",
+      "got_raw": "104",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "68",
+      "got": "108",
+      "got_raw": "108",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "180",
+      "got": "7",
+      "got_raw": "7",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "148",
+      "got": "148",
+      "got_raw": "148",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "298",
+      "got": "148",
+      "got_raw": "148",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "10000",
+      "got": "10000",
+      "got_raw": "10000",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "'txid'",
+      "got": "'txid'",
+      "got_raw": "'txid'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "'script'",
+      "got": "'script'",
+      "got_raw": "'script'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "unspent2",
+      "got": "unspent2",
+      "got_raw": "unspent2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "unspent3",
+      "got": "unspent3",
+      "got_raw": "unspent3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "Unspent.from_dict(unspent.to_dict())",
+      "got": "{",
+      "got_raw": "{ 'amount': 10000, 'height': 7, 'script': 'script', 'txid': 'txid', 'index': 0, }",
+      "exact_match": false,
+      "code_bleu": 0.05,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "\"Unspent(amount=10000, confirmations=7, \" \"script='script', txid='txid', txindex=0, \" \"segwit=False, \" \"sequence=4294967295)\"",
+      "got": "\"Unspent(10000, 7, 'script', 'txid', 0)\"",
+      "got_raw": "\"Unspent(10000, 7, 'script', 'txid', 0)\"",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.48484848484848486
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "30",
+      "got": "30",
+      "got_raw": "30",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "100",
+      "got": "100000000",
+      "got_raw": "100000000",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "2000",
+      "got": "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+      "got_raw": "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.045454545454545456
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "100000",
+      "got": "100000000",
+      "got_raw": "100000000",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "'0.01'",
+      "got": "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+      "got_raw": "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.04477611940298507
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "updated",
+      "got": "updated",
+      "got_raw": "updated",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "100000000",
+      "got": "100000000",
+      "got_raw": "100000000",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "cached_time",
+      "got": "cached_time",
+      "got_raw": "cached_time",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "'0.00000001'",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.15384615384615385
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "currency_to_satoshi(1, 'jpy')",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "1",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "ofek/bit",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "100",
+      "got": "100",
+      "got_raw": "100",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "1024",
+      "got": "1024",
+      "got_raw": "1024",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "2048",
+      "got": "2048",
+      "got_raw": "2048",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"tool1\"",
+      "got": "\"tool1\"",
+      "got_raw": "\"tool1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"server2\"",
+      "got": "\"server2\"",
+      "got_raw": "\"server2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"prompt1\"",
+      "got": "\"prompt1\"",
+      "got_raw": "\"prompt1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "event.raw_request",
+      "got": "event.raw_request",
+      "got_raw": "event.raw_request",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"TOOL_INVOCATION\"",
+      "got": "AccessEventType.TOOL_INVOCATION",
+      "got_raw": "AccessEventType.TOOL_INVOCATION",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.625
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"npx\"",
+      "got": "\"npx\"",
+      "got_raw": "\"npx\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "result.output",
+      "got": "result.output",
+      "got_raw": "result.output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"mcpm\"",
+      "got": "\"mcpm_test-server\"",
+      "got_raw": "\"mcpm_test-server\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"Windsurf\")",
+      "got": "\"test\")",
+      "got_raw": "\"test\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"npx\")",
+      "got": "\"npx\")",
+      "got_raw": "\"npx\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"bunx\")",
+      "got": "\"bunx\")",
+      "got_raw": "\"bunx\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "result.output",
+      "got": "result.output",
+      "got_raw": "result.output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "0",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "result.output",
+      "got": "result.output",
+      "got_raw": "result.output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"node\"",
+      "got": "\"node\"",
+      "got_raw": "\"node\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"python\"",
+      "got": "\"python\"",
+      "got_raw": "\"python\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "[\"server.js\"]",
+      "got": "[\"server.js\"]",
+      "got_raw": "[\"server.js\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "middleware_types",
+      "got": "middleware_types",
+      "got_raw": "middleware_types",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "config.mcpServers",
+      "got": "config.mcpServers",
+      "got_raw": "config.mcpServers",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "[\"-m\", \"test_server\"]",
+      "got": "[\"-m\", \"test_server\"]",
+      "got_raw": "[\"-m\", \"test_server\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"https://api.example.com/mcp\"",
+      "got": "\"https://api.example.com/mcp\"",
+      "got_raw": "\"https://api.example.com/mcp\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "{\"Authorization\": \"Bearer token123\"}",
+      "got": "{\"Authorization\": \"Bearer token123\"}",
+      "got_raw": "{\"Authorization\": \"Bearer token123\"}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "ValueError, match=\"No supported servers to proxy\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.36065573770491804
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "{\"MCPM_STDIO_SERVER\": \"true\", \"API_KEY\": \"test123\"}",
+      "got": "{\"API_KEY\": \"test123\"}",
+      "got_raw": "{\"API_KEY\": \"test123\"}",
+      "exact_match": false,
+      "code_bleu": 0.44181707153725036,
+      "edit_similarity": 0.6027397260273972
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "{}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "result.output",
+      "got": "result.output",
+      "got_raw": "result.output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "result.output.lower()",
+      "got": "result.output",
+      "got_raw": "result.output",
+      "exact_match": false,
+      "code_bleu": 0.36038417796587763,
+      "edit_similarity": 0.7647058823529411
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "result.output",
+      "got": "result.output",
+      "got_raw": "result.output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"nonexistent\")",
+      "got": "\"nonexistent\")",
+      "got_raw": "\"nonexistent\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"test-server\")",
+      "got": "\"test-server\")",
+      "got_raw": "\"test-server\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "3",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "stashed",
+      "got": "stashed",
+      "got_raw": "stashed",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "profiles",
+      "got": "profiles",
+      "got_raw": "profiles",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"%web-dev\"",
+      "got": "\"%web-dev\"",
+      "got_raw": "\"%web-dev\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "backup_names",
+      "got": "backup_names",
+      "got_raw": "backup_names",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "readme_content",
+      "got": "readme_content",
+      "got_raw": "readme_content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "stashed[\"claude-desktop\"]",
+      "got": "stashed[\"claude-desktop\"]",
+      "got_raw": "stashed[\"claude-desktop\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "analysis[\"clients_with_stashed\"]",
+      "got": "analysis[\"stashed_servers\"]",
+      "got_raw": "analysis[\"stashed_servers\"]",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.6440677966101694
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "[cmd.name for cmd in main.commands.values()]",
+      "got": "main.commands",
+      "got_raw": "main.commands",
+      "exact_match": false,
+      "code_bleu": 0.16817928305074292,
+      "edit_similarity": 0.45614035087719296
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "result.output",
+      "got": "result.output",
+      "got_raw": "result.output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "{}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "1",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "[0, 1]",
+      "got": "[0, 1]",
+      "got_raw": "[0, 1]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"server2\")",
+      "got": "server2)",
+      "got_raw": "server2)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "result.output",
+      "got": "result.output",
+      "got_raw": "result.output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "result.output",
+      "got": "result.output",
+      "got_raw": "result.output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "5",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "130",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"run\")",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"ERROR\"",
+      "got": "\"INFO\"",
+      "got_raw": "\"INFO\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.46153846153846156
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"WARNING\"",
+      "got": "\"INFO\"",
+      "got_raw": "\"INFO\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5333333333333333
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "caplog.text",
+      "got": "result.output",
+      "got_raw": "result.output",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "None)",
+      "got": "None)",
+      "got_raw": "None)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"test\")",
+      "got": "query=\"test\")",
+      "got_raw": "query=\"test\")",
+      "exact_match": false,
+      "code_bleu": 0.47287080450158786,
+      "edit_similarity": 0.7
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"github\")",
+      "got": "[\"github\"])",
+      "got_raw": "[\"github\"])",
+      "exact_match": false,
+      "code_bleu": 0.3976353643835253,
+      "edit_similarity": 0.9
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "result.output",
+      "got": "result.output",
+      "got_raw": "result.output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "9999",
+      "got": "9999",
+      "got_raw": "9999",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"global\"",
+      "got": "\"/tmp\"",
+      "got_raw": "\"/tmp\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "mock_config",
+      "got": "mock_config",
+      "got_raw": "mock_config",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "result.output",
+      "got": "result.output",
+      "got_raw": "result.output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "profiles",
+      "got": "profiles",
+      "got_raw": "profiles",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"server-1\"",
+      "got": "\"server-1\"",
+      "got_raw": "\"server-1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"test-server\"",
+      "got": "\"test-server\"",
+      "got_raw": "\"test-server\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"sk-test-123\"",
+      "got": "\"sk-test-123\"",
+      "got_raw": "\"sk-test-123\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"shared-server\"",
+      "got": "\"shared-server\"",
+      "got_raw": "\"shared-server\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"sk-complete-123\"",
+      "got": "\"sk-complete-123\"",
+      "got_raw": "\"sk-complete-123\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "servers",
+      "got": "servers",
+      "got_raw": "servers",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"test-server\"",
+      "got": "\"test-server\"",
+      "got_raw": "\"test-server\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"stdio-server\"",
+      "got": "\"stdio-server\"",
+      "got_raw": "\"stdio-server\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"computercontroller\"",
+      "got": "\"computercontroller\"",
+      "got_raw": "\"computercontroller\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "{}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "config",
+      "got": "config",
+      "got_raw": "config",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"qwen\")",
+      "got": "\"qwen\")",
+      "got_raw": "\"qwen\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"openai\"",
+      "got": "\"Qwen\"",
+      "got_raw": "\"Qwen\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"qwen-cli\"",
+      "got": "\"client_key\"",
+      "got_raw": "\"client_key\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.45454545454545453
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"Qwen CLI\"",
+      "got": "\"Qwen\"",
+      "got_raw": "\"Qwen\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"npx\"",
+      "got": "\"npx\"",
+      "got_raw": "\"npx\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "config",
+      "got": "config",
+      "got_raw": "config",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"npx\"",
+      "got": "\"npx\"",
+      "got_raw": "\"npx\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "servers",
+      "got": "servers",
+      "got_raw": "servers",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "server_list",
+      "got": "[server.name",
+      "got_raw": "[server.name for server in server_list]",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.5217391304347826
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"sample-key\"",
+      "got": "sample_server_config.env[\"API_KEY\"]",
+      "got_raw": "sample_server_config.env[\"API_KEY\"]",
+      "exact_match": false,
+      "code_bleu": 0.18575057999133598,
+      "edit_similarity": 0.3404255319148936
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"test-value\"",
+      "got": "\"test-value\"",
+      "got_raw": "\"test-value\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pathintegral-institute/mcpm.sh",
+      "expected": "\"test-server\"",
+      "got": "\"test-server\"",
+      "got_raw": "\"test-server\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "fn(x) - c)",
+      "got": "fn(x) - c)",
+      "got_raw": "fn(x) - c)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "fn(torch.full_like(x, c)))",
+      "got": "fn(torch.full_like(x, c)))",
+      "got_raw": "fn(torch.full_like(x, c)))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "alpha * y1 + (1 - alpha) * y2",
+      "got": "alpha * y1 + (1 - alpha) * y2",
+      "got_raw": "alpha * y1 + (1 - alpha) * y2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "parser[\"tool.poetry\"][\"version\"].replace('\"', \"\")",
+      "got": "parser[\"tool.poetry\"].get(\"version\")",
+      "got_raw": "parser[\"tool.poetry\"].get(\"version\")",
+      "exact_match": false,
+      "code_bleu": 0.30927711156292337,
+      "edit_similarity": 0.7294117647058823
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "targets)",
+      "got": "0.0)",
+      "got_raw": "0.0)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "RuntimeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "spot)",
+      "got": "spot)",
+      "got_raw": "spot)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "volatility)",
+      "got": "variance)",
+      "got_raw": "variance)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "time_to_maturity)",
+      "got": "time_to_maturity)",
+      "got_raw": "time_to_maturity)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "dtype",
+      "got": "dtype",
+      "got_raw": "dtype",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "f[0])",
+      "got": "0.0)",
+      "got_raw": "0.0)",
+      "exact_match": false,
+      "code_bleu": 0.2149139863647084,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "\"spot\"",
+      "got": "\"Spot()\"",
+      "got_raw": "\"Spot()\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7142857142857143
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "\"zeros\"",
+      "got": "\"Zeros()\"",
+      "got_raw": "\"Zeros()\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "\"empty\"",
+      "got": "\"Empty()\"",
+      "got_raw": "\"Empty()\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "device)",
+      "got": "device)",
+      "got_raw": "device)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "KeyError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.7
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "TypeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "feature.__class__",
+      "got": "feature.__class__",
+      "got_raw": "feature.__class__",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "dtype",
+      "got": "dtype",
+      "got_raw": "dtype",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "DeprecationWarning)",
+      "got": "ValueError",
+      "got_raw": "ValueError, match=\".*dtype.*\")",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.13793103448275862
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.device(device)",
+      "got": "device",
+      "got_raw": "device",
+      "exact_match": false,
+      "code_bleu": 0.09910059291689341,
+      "edit_similarity": 0.46153846153846156
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "european.payoff())",
+      "got": "european.payoff())",
+      "got_raw": "european.payoff())",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.tensor([2.0, 1.0, 0.0]).to(device))",
+      "got": "torch.tensor([0.0, 0.0, 0.0]))",
+      "got_raw": "torch.tensor([0.0, 0.0, 0.0]))",
+      "exact_match": false,
+      "code_bleu": 0.41392176829569666,
+      "edit_similarity": 0.7887323943661971
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.tensor([1.0, 2.0, 5.0]).to(device))",
+      "got": "torch.tensor([0.0, 0.0, 0.0]).to(device))",
+      "got_raw": "torch.tensor([0.0, 0.0, 0.0]).to(device))",
+      "exact_match": false,
+      "code_bleu": 0.6374542946641344,
+      "edit_similarity": 0.926829268292683
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "dtype",
+      "got": "dtype",
+      "got_raw": "dtype",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "spot)",
+      "got": "spot)",
+      "got_raw": "spot)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "dtype",
+      "got": "dtype",
+      "got_raw": "dtype",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "DeprecationWarning)",
+      "got": "ValueError",
+      "got_raw": "ValueError, match=\".*dtype.*\")",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.13793103448275862
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.device(device)",
+      "got": "device",
+      "got_raw": "device",
+      "exact_match": false,
+      "code_bleu": 0.09910059291689341,
+      "edit_similarity": 0.46153846153846156
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "dtype",
+      "got": "dtype",
+      "got_raw": "dtype",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "DeprecationWarning)",
+      "got": "ValueError",
+      "got_raw": "ValueError, match=\".*deprecated.*\")",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.13793103448275862
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "DeprecationWarning)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "DeprecationWarning)",
+      "got": "DeprecationWarning)",
+      "got_raw": "DeprecationWarning)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "dtype",
+      "got": "dtype",
+      "got_raw": "dtype",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "KeyError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.7
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "TypeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((10, 3))",
+      "got": "(10,)",
+      "got_raw": "(10,)",
+      "exact_match": false,
+      "code_bleu": 0.282842712474619,
+      "edit_similarity": 0.4166666666666667
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((10, 4))",
+      "got": "torch.Size((10, 4))",
+      "got_raw": "torch.Size((10, 4))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((10, 3))",
+      "got": "(10,)",
+      "got_raw": "(10,)",
+      "exact_match": false,
+      "code_bleu": 0.282842712474619,
+      "edit_similarity": 0.4166666666666667
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((10, 4))",
+      "got": "torch.Size((10, 4))",
+      "got_raw": "torch.Size((10, 4))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.ones_like(stock.spot))",
+      "got": "torch.zeros_like(stock.spot))",
+      "got_raw": "torch.zeros_like(stock.spot))",
+      "exact_match": false,
+      "code_bleu": 0.7071067811865475,
+      "edit_similarity": 0.9122807017543859
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.zeros_like(stock.volatility))",
+      "got": "torch.zeros_like(stock.volatility))",
+      "got_raw": "torch.zeros_like(stock.volatility))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((10, 3))",
+      "got": "(10,)",
+      "got_raw": "(10,)",
+      "exact_match": false,
+      "code_bleu": 0.282842712474619,
+      "edit_similarity": 0.4166666666666667
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((10, 4))",
+      "got": "torch.Size((10, 4))",
+      "got_raw": "torch.Size((10, 4))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((10, 3))",
+      "got": "(10,)",
+      "got_raw": "(10,)",
+      "exact_match": false,
+      "code_bleu": 0.282842712474619,
+      "edit_similarity": 0.4166666666666667
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((10, 4))",
+      "got": "torch.Size((10, 4))",
+      "got_raw": "torch.Size((10, 4))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((10, 3))",
+      "got": "(10,)",
+      "got_raw": "(10,)",
+      "exact_match": false,
+      "code_bleu": 0.282842712474619,
+      "edit_similarity": 0.4166666666666667
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((10, 4))",
+      "got": "torch.Size((10, 4))",
+      "got_raw": "torch.Size((10, 4))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "i1)",
+      "got": "i1)",
+      "got_raw": "i1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "i2)",
+      "got": "i2)",
+      "got_raw": "i2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "i3)",
+      "got": "i3)",
+      "got_raw": "i3)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "i4)",
+      "got": "i4)",
+      "got_raw": "i4)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "-payoff)",
+      "got": "-payoff)",
+      "got_raw": "-payoff)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "TypeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N,))",
+      "got": "(N,)",
+      "got_raw": "(N,)",
+      "exact_match": false,
+      "code_bleu": 0.4444444444444444,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, H, T))",
+      "got": "(N, H)",
+      "got_raw": "(N, H)",
+      "exact_match": false,
+      "code_bleu": 0.3313628036529378,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, 1, h_in))",
+      "got": "(N, T)",
+      "got_raw": "(N, T)",
+      "exact_match": false,
+      "code_bleu": 0.22386873579657127,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "a)",
+      "got": "a)",
+      "got_raw": "a)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "c)",
+      "got": "device)",
+      "got_raw": "device)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "x2)",
+      "got": "x2)",
+      "got_raw": "x2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "eta)",
+      "got": "device)",
+      "got_raw": "device)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "loss)",
+      "got": "loss)",
+      "got_raw": "loss)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "in_expect",
+      "got": "in_expect",
+      "got_raw": "in_expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "out_expect",
+      "got": "out_expect",
+      "got_raw": "out_expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "out_features",
+      "got": "out_features",
+      "got_raw": "out_features",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "2 * (n_layers + 1)",
+      "got": "n_layers",
+      "got_raw": "n_layers",
+      "exact_match": false,
+      "code_bleu": 0.08494336535733721,
+      "edit_similarity": 0.6153846153846154
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, H_out))",
+      "got": "(N, H_out)",
+      "got_raw": "(N, H_out)",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.625
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, M_1, H_out))",
+      "got": "torch.Size((N, M_2, H_out))",
+      "got_raw": "torch.Size((N, M_2, H_out))",
+      "exact_match": false,
+      "code_bleu": 0.7611606003349892,
+      "edit_similarity": 0.9629629629629629
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, M_1, M_2, H_out))",
+      "got": "torch.Size((N, M_1, M_2, H_out))",
+      "got_raw": "torch.Size((N, M_1, M_2, H_out))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, H_out))",
+      "got": "(N, H_out)",
+      "got_raw": "(N, H_out)",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.625
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, M_1, H_out))",
+      "got": "torch.Size((N, M_2, H_out))",
+      "got_raw": "torch.Size((N, M_2, H_out))",
+      "exact_match": false,
+      "code_bleu": 0.7611606003349892,
+      "edit_similarity": 0.9629629629629629
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, M_1, M_2, H_out))",
+      "got": "torch.Size((N, M_1, M_2, H_out))",
+      "got_raw": "torch.Size((N, M_1, M_2, H_out))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "\"SVIVariance(a=1.0, b=0, rho=0.1, m=0.2, sigma=0.3)\"",
+      "got": "\"SVIVariance(a=1.0, b=0, rho=0.1, m=0.2, sigma=0.3)\"",
+      "got_raw": "\"SVIVariance(a=1.0, b=0, rho=0.1, m=0.2, sigma=0.3)\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, 1))",
+      "got": "(N, M_1)",
+      "got_raw": "(N, M_1)",
+      "exact_match": false,
+      "code_bleu": 0.2686424829558855,
+      "edit_similarity": 0.46153846153846156
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, M_1, 1))",
+      "got": "torch.Size((N, M_2))",
+      "got_raw": "torch.Size((N, M_2))",
+      "exact_match": false,
+      "code_bleu": 0.6182872189470843,
+      "edit_similarity": 0.8837209302325582
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, M_1, M_2, 1))",
+      "got": "torch.Size((N, M_1, M_2))",
+      "got_raw": "torch.Size((N, M_1, M_2))",
+      "exact_match": false,
+      "code_bleu": 0.7543814886800514,
+      "edit_similarity": 0.9433962264150944
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N,))",
+      "got": "(N,",
+      "got_raw": "(N, M_1)",
+      "exact_match": false,
+      "code_bleu": 0.2802988050845715,
+      "edit_similarity": 0.3157894736842105
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, 1))",
+      "got": "torch.Size((N,))",
+      "got_raw": "torch.Size((N,))",
+      "exact_match": false,
+      "code_bleu": 0.7476316626971513,
+      "edit_similarity": 0.9411764705882353
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, M_1))",
+      "got": "torch.Size((N, M_2))",
+      "got_raw": "torch.Size((N, M_2))",
+      "exact_match": false,
+      "code_bleu": 0.7419446627365011,
+      "edit_similarity": 0.95
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, M_1, 1))",
+      "got": "torch.Size((N, M_1))",
+      "got_raw": "torch.Size((N, M_1))",
+      "exact_match": false,
+      "code_bleu": 0.7093467443268724,
+      "edit_similarity": 0.9302325581395349
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, M_1, M_2))",
+      "got": "torch.Size((N, M_1, M_2))",
+      "got_raw": "torch.Size((N, M_1, M_2))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, M_1, M_2, 1))",
+      "got": "torch.Size((N, M_1, M_2, 1))",
+      "got_raw": "torch.Size((N, M_1, M_2, 1))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "m)",
+      "got": "m)",
+      "got_raw": "m)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "1.0",
+      "got": "1.0",
+      "got_raw": "1.0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "2.0",
+      "got": "2.0",
+      "got_raw": "2.0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "BSEuropeanOption",
+      "got": "BlackScholes",
+      "got_raw": "BlackScholes",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.21428571428571427
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, 1))",
+      "got": "(N, M_1)",
+      "got_raw": "(N, M_1)",
+      "exact_match": false,
+      "code_bleu": 0.2686424829558855,
+      "edit_similarity": 0.46153846153846156
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, M_1, 1))",
+      "got": "torch.Size((N, M_2))",
+      "got_raw": "torch.Size((N, M_2))",
+      "exact_match": false,
+      "code_bleu": 0.6182872189470843,
+      "edit_similarity": 0.8837209302325582
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((N, M_1, M_2, 1))",
+      "got": "torch.Size((N, M_1, M_2, 1))",
+      "got_raw": "torch.Size((N, M_1, M_2, 1))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "m)",
+      "got": "m)",
+      "got_raw": "m)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "m)",
+      "got": "m)",
+      "got_raw": "m)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "price)",
+      "got": "price)",
+      "got_raw": "price)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "m)",
+      "got": "m)",
+      "got_raw": "m)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.float32",
+      "got": "torch.float32",
+      "got_raw": "torch.float32",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.float64",
+      "got": "torch.float64",
+      "got_raw": "torch.float64",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((n_paths, n_steps))",
+      "got": "(n_paths, n_steps)",
+      "got_raw": "(n_paths, n_steps)",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.float32",
+      "got": "torch.float32",
+      "got_raw": "torch.float32",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.float64",
+      "got": "torch.float64",
+      "got_raw": "torch.float64",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "output.variance.sqrt())",
+      "got": "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+      "got_raw": "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+      "exact_match": false,
+      "code_bleu": 0.0625,
+      "edit_similarity": 0.013245033112582781
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "ValueError, match=\"jump_mean_down must be postive\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.3548387096774194
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "ValueError, match=\"jump prob must be in 0 and 1 incl\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.3384615384615385
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "ValueError, match=\"jump_mean_up must be postive and smaller than 1\")",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect)",
+      "got": "expect)",
+      "got_raw": "expect)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.float32",
+      "got": "torch.float32",
+      "got_raw": "torch.float32",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.float64",
+      "got": "torch.float64",
+      "got_raw": "torch.float64",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.Size((n_paths, n_steps))",
+      "got": "(n_paths, n_steps)",
+      "got_raw": "(n_paths, n_steps)",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "torch.zeros_like(output[0]))",
+      "got": "0.0)",
+      "got_raw": "0.0)",
+      "exact_match": false,
+      "code_bleu": 0.1074569931823542,
+      "edit_similarity": 0.125
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "expect",
+      "got": "expect",
+      "got_raw": "expect",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "iv_expected)",
+      "got": "iv_expected)",
+      "got_raw": "iv_expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pfnet-research/pfhedge",
+      "expected": "output.variance.sqrt())",
+      "got": "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+      "got_raw": "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+      "exact_match": false,
+      "code_bleu": 0.0625,
+      "edit_similarity": 0.013245033112582781
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "obj",
+      "got": "obj",
+      "got_raw": "obj",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "str",
+      "got": "str",
+      "got_raw": "str",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "int",
+      "got": "int",
+      "got_raw": "int",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "env",
+      "got": "env",
+      "got_raw": "env",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "val2",
+      "got": "val2",
+      "got_raw": "val2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "impl",
+      "got": "impl",
+      "got_raw": "impl",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "output",
+      "got": "output",
+      "got_raw": "output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "result",
+      "got": "result",
+      "got_raw": "result",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "result",
+      "got": "result",
+      "got_raw": "result",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "content",
+      "got": "content",
+      "got_raw": "content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "content",
+      "got": "content",
+      "got_raw": "content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "ts_code",
+      "got": "ts_code",
+      "got_raw": "ts_code",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "ts_type",
+      "got": "ts_type",
+      "got_raw": "ts_type",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "ts_code",
+      "got": "ts_code",
+      "got_raw": "ts_code",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "ts_code",
+      "got": "ts_code",
+      "got_raw": "ts_code",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "ts_code",
+      "got": "ts_code",
+      "got_raw": "ts_code",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "ts_code",
+      "got": "ts_code",
+      "got_raw": "ts_code",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "min_time",
+      "got": "min_time",
+      "got_raw": "min_time",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "max_time",
+      "got": "max_time",
+      "got_raw": "max_time",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "view_base",
+      "got": "view_base",
+      "got_raw": "view_base",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "log_level",
+      "got": "log_level",
+      "got_raw": "log_level",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "root_path",
+      "got": "root_path",
+      "got_raw": "root_path",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "error_str",
+      "got": "error_str",
+      "got_raw": "error_str",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "child_idx",
+      "got": "child_idx",
+      "got_raw": "child_idx",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "responses",
+      "got": "responses",
+      "got_raw": "responses",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "log_output",
+      "got": "log_output",
+      "got_raw": "log_output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "TypeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "TypeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "lock_second",
+      "got": "lock_second",
+      "got_raw": "lock_second",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "logger_name",
+      "got": "logger_name",
+      "got_raw": "logger_name",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "mock_session",
+      "got": "mock_session",
+      "got_raw": "mock_session",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "test_content",
+      "got": "test_content",
+      "got_raw": "test_content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "expected_url",
+      "got": "expected_url",
+      "got_raw": "expected_url",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "ExampleModel",
+      "got": "str",
+      "got_raw": "str",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "expected_str",
+      "got": "expected_str",
+      "got_raw": "expected_str",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "expected_copy",
+      "got": "expected_copy",
+      "got_raw": "expected_copy",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "expected_tags",
+      "got": "expected_tags",
+      "got_raw": "expected_tags",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "TimeoutError)",
+      "got": "V8RuntimeError)",
+      "got_raw": "V8RuntimeError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6428571428571429
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "response_data",
+      "got": "response_data",
+      "got_raw": "response_data",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "error_payload",
+      "got": "error_payload",
+      "got_raw": "error_payload",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "response_data",
+      "got": "response_data",
+      "got_raw": "response_data",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "main_model_idx",
+      "got": "main_model_idx",
+      "got_raw": "main_model_idx",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "expected_prefix",
+      "got": "expected_prefix",
+      "got_raw": "expected_prefix",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "child_model_idx",
+      "got": "child_model_idx",
+      "got_raw": "child_model_idx",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "expected_ignore",
+      "got": "expected_ignore",
+      "got_raw": "expected_ignore",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "ValidationError)",
+      "got": "ValidationError)",
+      "got_raw": "ValidationError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "expected_is_file",
+      "got": "expected_is_file",
+      "got_raw": "expected_is_file",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "args[0]",
+      "got": "args[0]",
+      "got_raw": "args[0]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "args[1]",
+      "got": "args[1]",
+      "got_raw": "args[1]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "args[2]",
+      "got": "args[2]",
+      "got_raw": "args[2]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "result[0]",
+      "got": "result",
+      "got_raw": "result",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "captured.out",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "result.stdout",
+      "got": "result.stdout",
+      "got_raw": "result.stdout",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "tmp_path / \"views\"",
+      "got": "tmp_path / \"views\"",
+      "got_raw": "tmp_path / \"views\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "root_path / \"_ssr\"",
+      "got": "root_path / \"_ssr\"",
+      "got_raw": "root_path / \"_ssr\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "expected[key].alias",
+      "got": "expected[key].alias",
+      "got_raw": "expected[key].alias",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "interface.typehints",
+      "got": "interface.params",
+      "got_raw": "interface.params",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.6857142857142857
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "CallbackType.CREATED",
+      "got": "CallbackType.add",
+      "got_raw": "CallbackType.add",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.7222222222222222
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "CallbackType.DELETED",
+      "got": "CallbackType.DELETED",
+      "got_raw": "CallbackType.DELETED",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "root_path / \"_server\"",
+      "got": "Path(tmpdir)",
+      "got_raw": "Path(tmpdir)",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.24242424242424243
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "root_path / \"_static\"",
+      "got": "root_path / \"_static\"",
+      "got_raw": "root_path / \"_static\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "expected[key].default",
+      "got": "expected[key].default",
+      "got_raw": "expected[key].default",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "CallbackType.MODIFIED",
+      "got": "Change.modified",
+      "got_raw": "Change.modified",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "log_output[\"exception\"]",
+      "got": "log_output[\"exception\"]",
+      "got_raw": "log_output[\"exception\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "interface.response_type",
+      "got": "interface.typehints",
+      "got_raw": "interface.typehints",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "session_cache.loop_locks",
+      "got": "session_cache",
+      "got_raw": "session_cache",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.7027027027027027
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "expected[key].annotation",
+      "got": "expected[key].annotation",
+      "got_raw": "expected[key].annotation",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "result_model.model_fields",
+      "got": "result_model.__fields__",
+      "got_raw": "result_model.__fields__",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.8333333333333334
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "root_path / \"detail/page.tsx\"",
+      "got": "\"/detail/page.tsx\"",
+      "got_raw": "\"/detail/page.tsx\"",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.723404255319149
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "metadata.static_artifact_shas",
+      "got": "metadata",
+      "got_raw": "metadata",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.43243243243243246
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "FunctionActionType.PASSTHROUGH",
+      "got": "FunctionActionType.PASSTHROUGH",
+      "got_raw": "FunctionActionType.PASSTHROUGH",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "interface.include_superclasses",
+      "got": "interface.base_classes",
+      "got_raw": "interface.base_classes",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.7307692307692307
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "ssr_base / \"stub_controller.js\"",
+      "got": "ssr_base",
+      "got_raw": "ssr_base",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.41025641025641024
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "package_dir_path / package_name",
+      "got": "package_dir_path",
+      "got_raw": "package_dir_path",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.6808510638297872
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "TypeError, match=\"1 is not a string\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.2916666666666667
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "[\"age\"]",
+      "got": "\"age\"",
+      "got_raw": "\"age\"",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.8333333333333334
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "{\"a\": 4}",
+      "got": "{\"a\": [2",
+      "got_raw": "{\"a\": [2, 3, 5, 7, 11, 13, 17, 19, 23, 29]}",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "{\"a\": 0}",
+      "got": "{\"a\": 0}",
+      "got_raw": "{\"a\": 0}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "{\"a\": 20}",
+      "got": "ExampleResponse(a=10)",
+      "got_raw": "ExampleResponse(a=10)",
+      "exact_match": false,
+      "code_bleu": 0.18575057999133598,
+      "edit_similarity": 0.13333333333333333
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "{\"fastapi\"}",
+      "got": "{",
+      "got_raw": "{ \"fasta\", \"fasta-1.0.0\", \"fasta-1.0.0-py3-none-any\", \"fasta-1.0.0-py3.10-none-any\", \"fasta-1.0.0-py3.11-none-any\", \"fasta-1.0.0-py3.12-none-any\", \"fasta-1.0.0-py3.13-none-any\", \"fasta-1.0.0-py3.14-none-any\", \"fasta-1.0.0-py3.15-none-any",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "{\"a\": 1244}",
+      "got": "{\"a\": 1244}",
+      "got_raw": "{\"a\": 1244}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "(\"test_webapp\",)",
+      "got": "()",
+      "got_raw": "()",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "{\"resolved_dep\"}",
+      "got": "{\"payload\"",
+      "got_raw": "{\"payload\", \"request\"}",
+      "exact_match": false,
+      "code_bleu": 0.35818997727451396,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "{\"job1\", \"job2\"}",
+      "got": "{\"job1\", \"job2\"}",
+      "got_raw": "{\"job1\", \"job2\"}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "(\"line1\", \"line2\")",
+      "got": "[\"line1\", \"line2\"]",
+      "got_raw": "[\"line1\", \"line2\"]",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "{\"resolved_dep\": 1}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.23784142300054423,
+      "edit_similarity": 0.19047619047619047
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "{\"a\": 1244, \"b\": 78503}",
+      "got": "optimized_func()",
+      "got_raw": "optimized_func()",
+      "exact_match": false,
+      "code_bleu": 0.12663928094193208,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "{\"raw_value\": \"success\"}",
+      "got": "{\"raw_value\": \"success\"}",
+      "got_raw": "{\"raw_value\": \"success\"}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "(None, ResponseModelType.SINGLE_RESPONSE)",
+      "got": "\"ExampleModel\"",
+      "got_raw": "\"ExampleModel\"",
+      "exact_match": false,
+      "code_bleu": 0.07142857142857142,
+      "edit_similarity": 0.2545454545454545
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "(ExampleModel, ResponseModelType.SINGLE_RESPONSE)",
+      "got": "ExampleModel",
+      "got_raw": "ExampleModel",
+      "exact_match": false,
+      "code_bleu": 0.08494336535733721,
+      "edit_similarity": 0.39344262295081966
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "{ T: int, K: str, }",
+      "got": "{T: Final}",
+      "got_raw": "{T: Final}",
+      "exact_match": false,
+      "code_bleu": 0.2686424829558855,
+      "edit_similarity": 0.4827586206896552
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "{ K: int, T: int, }",
+      "got": "[()]",
+      "got_raw": "[()]",
+      "exact_match": false,
+      "code_bleu": 0.12085501589427072,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "{ \"pydantic\", \"pydantic_core\", \"pydantic-settings\", }",
+      "got": "{\"pydan\"}",
+      "got_raw": "{\"pydan\"}",
+      "exact_match": false,
+      "code_bleu": 0.18749999999999997,
+      "edit_similarity": 0.2903225806451613
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "{ str(Path(path).absolute()) for path in expected_paths }",
+      "got": "set(expected_paths)",
+      "got_raw": "set(expected_paths)",
+      "exact_match": false,
+      "code_bleu": 0.10053811915205507,
+      "edit_similarity": 0.4473684210526316
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "[ CallbackType.CREATED, CallbackType.MODIFIED, CallbackType.DELETED, ]",
+      "got": "[CallbackType.CREATED, CallbackType.MODIFIED, CallbackType.DELETED]",
+      "got_raw": "[CallbackType.CREATED, CallbackType.MODIFIED, CallbackType.DELETED]",
+      "exact_match": false,
+      "code_bleu": 0.8696199183331033,
+      "edit_similarity": 0.9781021897810219
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "{ \"project\": { \"name\": \"mountaineer\", \"version\": \"0.2.0\", \"description\": \"\", }, }",
+      "got": "{ \"project\": { \"name\": \"mountaineer\", \"version\": \"0.2.0\", }, }",
+      "got_raw": "{ \"project\": { \"name\": \"mountaineer\", \"version\": \"0.2.0\", }, }",
+      "exact_match": false,
+      "code_bleu": 0.7432823505379895,
+      "edit_similarity": 0.8671328671328671
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "{ \"tool\": { \"poetry\": { \"name\": \"mountaineer\", \"version\": \"0.2.0\", \"description\": \"\", }, }, }",
+      "got": "{ \"tool\": { \"poetry\": { \"name\": \"mountaineer\", \"version\": \"0.2.0\", }, }, }, }",
+      "got_raw": "{ \"tool\": { \"poetry\": { \"name\": \"mountaineer\", \"version\": \"0.2.0\", }, }, }, }",
+      "exact_match": false,
+      "code_bleu": 0.8055134165167693,
+      "edit_similarity": 0.8941176470588236
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "[ 'data: {\"passthrough\": {\"value\": \"Hello\"}}', 'data: {\"passthrough\": {\"value\": \"World\"}}', ]",
+      "got": "[b\"Hello, World!\\n\"]",
+      "got_raw": "[b\"Hello, World!\\n\"]",
+      "exact_match": false,
+      "code_bleu": 0.26509024292235017,
+      "edit_similarity": 0.2831858407079646
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "3",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "4",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "2",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "6",
+      "got": "6",
+      "got_raw": "6",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"/\"",
+      "got": "\"/\"",
+      "got_raw": "\"/\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"7\"",
+      "got": "7",
+      "got_raw": "7",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"8\"",
+      "got": "8",
+      "got_raw": "8",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"9\"",
+      "got": "9",
+      "got_raw": "9",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"abc\"",
+      "got": "\"abc\"",
+      "got_raw": "\"abc\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"def\"",
+      "got": "\"def\"",
+      "got_raw": "\"def\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"any\"",
+      "got": "\"Any\"",
+      "got_raw": "\"Any\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"null\"",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"0.1.0\"",
+      "got": "\"0.0.0\"",
+      "got_raw": "\"0.0.0\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"custom\"",
+      "got": "\"custom\"",
+      "got_raw": "\"custom\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"layout\"",
+      "got": "\"layout\"",
+      "got_raw": "\"layout\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"Status\"",
+      "got": "\"Status\"",
+      "got_raw": "\"Status\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"Test 402\"",
+      "got": "\"Test 402\"",
+      "got_raw": "\"Test 402\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"standard\"",
+      "got": "\"standard\"",
+      "got_raw": "\"standard\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"TestModel\"",
+      "got": "\"TestModel\"",
+      "got_raw": "\"TestModel\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"UserModel\"",
+      "got": "\"UserModel\"",
+      "got_raw": "\"UserModel\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"response1\"",
+      "got": "\"response1\"",
+      "got_raw": "\"response1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"response2\"",
+      "got": "\"response2\"",
+      "got_raw": "\"response2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"test-value\"",
+      "got": "\"test-value\"",
+      "got_raw": "\"test-value\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"StatusEnum\"",
+      "got": "\"StatusEnum\"",
+      "got_raw": "\"StatusEnum\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"int_parsing\"",
+      "got": "\"ValidationError\"",
+      "got_raw": "\"ValidationError\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"/test-page/\"",
+      "got": "\"/test-layout.tsx\"",
+      "got_raw": "\"/test-layout.tsx\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5806451612903226
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"/some-page/\"",
+      "got": "\"/some-page/\"",
+      "got_raw": "\"/some-page/\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"Set<number>\"",
+      "got": "\"Set[float]\"",
+      "got_raw": "\"Set[float]\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"SimpleModel\"",
+      "got": "\"SimpleModel\"",
+      "got_raw": "\"SimpleModel\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"not_a_number\"",
+      "got": "\"not_a_number\"",
+      "got_raw": "\"not_a_number\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"controller_a\"",
+      "got": "\"renders.a\"",
+      "got_raw": "\"renders.a\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.48
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"controller_b\"",
+      "got": "\"controller_b\"",
+      "got_raw": "\"controller_b\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"layout-value\"",
+      "got": "\"layout-value\"",
+      "got_raw": "\"layout-value\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"ContainerStr\"",
+      "got": "\"String\"",
+      "got_raw": "\"String\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.45454545454545453
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"ContainerInt\"",
+      "got": "\"ContainerInt\"",
+      "got_raw": "\"ContainerInt\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"don't do this\"",
+      "got": "\"7\"",
+      "got_raw": "\"7\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"simple_action\"",
+      "got": "\"simple_action\"",
+      "got_raw": "\"simple_action\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"Array<string>\"",
+      "got": "\"List[str]\"",
+      "got_raw": "\"List[str]\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.38461538461538464
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"CircularModel\"",
+      "got": "\"SimpleModel\"",
+      "got_raw": "\"SimpleModel\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6428571428571429
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"processed CSS\"",
+      "got": "\"processed CSS\"",
+      "got_raw": "\"processed CSS\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"late response\"",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3157894736842105
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"AuthModels_User\"",
+      "got": "\"AuthModels_User\"",
+      "got_raw": "\"AuthModels_User\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"TreeModels_Node\"",
+      "got": "\"Node\"",
+      "got_raw": "\"Node\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5217391304347826
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"ExampleException\"",
+      "got": "\"ExampleException\"",
+      "got_raw": "\"ExampleException\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"UsersModels_User\"",
+      "got": "User.__name__",
+      "got_raw": "User.__name__",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.3870967741935484
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"process-response\"",
+      "got": "\"process-response\"",
+      "got_raw": "\"process-response\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"get_external_data\"",
+      "got": "\"get_external_data\"",
+      "got_raw": "\"get_external_data\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"text/event-stream\"",
+      "got": "\"application/json\"",
+      "got_raw": "\"application/json\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.16216216216216217
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "\"session_test_value\"",
+      "got": "\"session_test_value\"",
+      "got_raw": "\"session_test_value\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "id(new_path)",
+      "got": "id(new_path)",
+      "got_raw": "id(new_path)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "set(expected)",
+      "got": "set(expected)",
+      "got_raw": "set(expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "expected.keys()",
+      "got": "expected.keys()",
+      "got_raw": "expected.keys()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "interface.to_js()",
+      "got": "interface.include_superclasses",
+      "got_raw": "interface.include_superclasses",
+      "exact_match": false,
+      "code_bleu": 0.3567621345008163,
+      "edit_similarity": 0.5106382978723404
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "interface.to_js()",
+      "got": "interface.fields",
+      "got_raw": "interface.fields",
+      "exact_match": false,
+      "code_bleu": 0.3567621345008163,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "str(HelloEnum.eight)",
+      "got": "8",
+      "got_raw": "8",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "error.message.lower()",
+      "got": "error.error",
+      "got_raw": "error.error",
+      "exact_match": false,
+      "code_bleu": 0.2548300960720116,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "str(GoodbyeEnum.nine)",
+      "got": "9",
+      "got_raw": "9",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "str(new_path.root_link)",
+      "got": "str(new_path.root_link)",
+      "got_raw": "str(new_path.root_link)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "expected[key].is_required()",
+      "got": "expected[key].is_required()",
+      "got_raw": "expected[key].is_required()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "str(new_path.package_root_link)",
+      "got": "str(new_path.package_root_link)",
+      "got_raw": "str(new_path.package_root_link)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "ExamplePassthroughModel(status=\"success\")",
+      "got": "\"passthrough\"",
+      "got_raw": "\"passthrough\"",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "piercefreeman/mountaineer",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "7",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "3",
+      "got": "0.0",
+      "got_raw": "0.0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "2",
+      "got": "1.0",
+      "got_raw": "1.0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "4)",
+      "got": "4)",
+      "got_raw": "4)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "2)",
+      "got": "2)",
+      "got_raw": "2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "7)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "6)",
+      "got": "10)",
+      "got_raw": "10)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0)",
+      "got": "6)",
+      "got_raw": "6)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "4)",
+      "got": "4.0)",
+      "got_raw": "4.0)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "7)",
+      "got": "5)",
+      "got_raw": "5)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "5)",
+      "got": "5)",
+      "got_raw": "5)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "5)",
+      "got": "4)",
+      "got_raw": "4)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.7",
+      "got": "0.85",
+      "got_raw": "0.85",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.9",
+      "got": "1.0",
+      "got_raw": "1.0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.4",
+      "got": "0.4",
+      "got_raw": "0.4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.7",
+      "got": "0.7",
+      "got_raw": "0.7",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.8",
+      "got": "0.8",
+      "got_raw": "0.8",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.9",
+      "got": "1.0",
+      "got_raw": "1.0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.0",
+      "got": "0.0",
+      "got_raw": "0.0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.2",
+      "got": "resid2",
+      "got_raw": "resid2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.3",
+      "got": "resid2",
+      "got_raw": "resid2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "11)",
+      "got": "9.0)",
+      "got_raw": "9.0)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.01",
+      "got": "-",
+      "got_raw": " - sa_new.levels[-1].A.toarray())))",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.25",
+      "got": "0.25",
+      "got_raw": "0.25",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.65",
+      "got": "1.0",
+      "got_raw": "1.0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.45",
+      "got": "0.5",
+      "got_raw": "0.5",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.85",
+      "got": "1.0",
+      "got_raw": "1.0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.01",
+      "got": "1e-10",
+      "got_raw": "1e-10",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.45",
+      "got": "0.45",
+      "got_raw": "0.45",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.92",
+      "got": "0.92",
+      "got_raw": "0.92",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.01",
+      "got": "1e-10",
+      "got_raw": "1e-10",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.01",
+      "got": "1e-10",
+      "got_raw": "1e-10",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.20",
+      "got": "1.0)",
+      "got_raw": "1.0)",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "1e-4",
+      "got": "1e-10",
+      "got_raw": "1e-10",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.0)",
+      "got": "0.0)",
+      "got_raw": "0.0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "1e-2",
+      "got": "tol",
+      "got_raw": "tol",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "1e-6",
+      "got": "1e-12",
+      "got_raw": "1e-12",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.0)",
+      "got": "0.0)",
+      "got_raw": "0.0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.0)",
+      "got": "0.0)",
+      "got_raw": "0.0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "1e-8",
+      "got": "0.5",
+      "got_raw": "0.5",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.01",
+      "got": "resid2",
+      "got_raw": "resid2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.03",
+      "got": "resid2",
+      "got_raw": "resid2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.95",
+      "got": "1.0",
+      "got_raw": "1.0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "1e-4",
+      "got": "1e-4",
+      "got_raw": "1e-4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "1e-7",
+      "got": "1e-8",
+      "got_raw": "1e-8",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "1.0)",
+      "got": "5",
+      "got_raw": "5 / 9.5)",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.0)",
+      "got": "1.0)",
+      "got_raw": "1.0)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.0)",
+      "got": "0.0)",
+      "got_raw": "0.0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.0)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "1e-12",
+      "got": "1e-12)",
+      "got_raw": "1e-12)",
+      "exact_match": false,
+      "code_bleu": 0.668740304976422,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0.54)",
+      "got": "0.545351286587159)",
+      "got_raw": "0.545351286587159)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.43478260869565216
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "n",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "B)",
+      "got": "B)",
+      "got_raw": "B)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "L)",
+      "got": "L)",
+      "got_raw": "L)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "M)",
+      "got": "M)",
+      "got_raw": "M)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "K)",
+      "got": "K)",
+      "got_raw": "K)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "B)",
+      "got": "B)",
+      "got_raw": "B)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "c)",
+      "got": "c)",
+      "got_raw": "c)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "b)",
+      "got": "b)",
+      "got_raw": "b)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "c)",
+      "got": "c)",
+      "got_raw": "c)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "V2)",
+      "got": "V2)",
+      "got_raw": "V2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "E2)",
+      "got": "E2)",
+      "got_raw": "E2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "AA)",
+      "got": "AA)",
+      "got_raw": "AA)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "bb)",
+      "got": "bb)",
+      "got_raw": "bb)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "tol",
+      "got": "tol",
+      "got_raw": "tol",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "x1)",
+      "got": "x1)",
+      "got_raw": "x1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "x2)",
+      "got": "x2)",
+      "got_raw": "x2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "x3)",
+      "got": "x3)",
+      "got_raw": "x3)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "A0)",
+      "got": "A0.toarray())",
+      "got_raw": "A0.toarray())",
+      "exact_match": false,
+      "code_bleu": 0.24446151121745047,
+      "edit_similarity": 0.375
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "pc)",
+      "got": "[1",
+      "got_raw": "[1, 1, 1, 1, 1, 1, 1, 1, 1])",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "Bf)",
+      "got": "B)",
+      "got_raw": "B)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "x64)",
+      "got": "x64)",
+      "got_raw": "x64)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "rhs)",
+      "got": "rhs)",
+      "got_raw": "rhs)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "x38)",
+      "got": "x38)",
+      "got_raw": "x38)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "mis)",
+      "got": "mis)",
+      "got_raw": "mis)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "D_A)",
+      "got": "D_A)",
+      "got_raw": "D_A)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "soln)",
+      "got": "soln)",
+      "got_raw": "soln)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "D_AA)",
+      "got": "D_AA)",
+      "got_raw": "D_AA)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "rratio",
+      "got": "1.0",
+      "got_raw": "1.0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "exact)",
+      "got": "exact)",
+      "got_raw": "exact)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "flag2)",
+      "got": "flag2)",
+      "got_raw": "flag2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "solver",
+      "got": "solver",
+      "got_raw": "solver",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "d_ref)",
+      "got": "d_ref)",
+      "got_raw": "d_ref)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "m_ref)",
+      "got": "m_ref)",
+      "got_raw": "m_ref)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "D_ref)",
+      "got": "D_ref)",
+      "got_raw": "D_ref)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "P_ref)",
+      "got": "P_ref)",
+      "got_raw": "P_ref)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "exact)",
+      "got": "exact)",
+      "got_raw": "exact)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "c_factor",
+      "got": "1.0",
+      "got_raw": "1.0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "expected",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.9411764705882353
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "c_factor",
+      "got": "1.0",
+      "got_raw": "1.0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "P_exact)",
+      "got": "P_exact)",
+      "got_raw": "P_exact)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "R_exact)",
+      "got": "R_exact)",
+      "got_raw": "R_exact)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "stencil)",
+      "got": "stencil=stencil))",
+      "got_raw": " stencil=stencil))",
+      "exact_match": false,
+      "code_bleu": 0.35930411196308426,
+      "edit_similarity": 0.64
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "scipyres)",
+      "got": "hhres[1:])",
+      "got_raw": "hhres[1:])",
+      "exact_match": false,
+      "code_bleu": 0.22089591134157885,
+      "edit_similarity": 0.42105263157894735
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "pinv_test)",
+      "got": "pinv_test)",
+      "got_raw": "pinv_test)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "A_expected)",
+      "got": "A_expected)",
+      "got_raw": "A_expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "B_expected)",
+      "got": "B_expected)",
+      "got_raw": "B_expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "d_expected)",
+      "got": "d_expected)",
+      "got_raw": "d_expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "m_expected)",
+      "got": "m_expected)",
+      "got_raw": "m_expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "expected_speed",
+      "got": "expected_speed)",
+      "got_raw": "expected_speed)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.9655172413793104
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "fine_candidates)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.1111111111111111
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "splitting_paper)",
+      "got": "splitting_paper)",
+      "got_raw": "splitting_paper)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "0*B)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "L + M)",
+      "got": "np.array([[4",
+      "got_raw": "np.array([[4, 1, -2, -1, -2, -1, 0, 1]]) / 4.0)",
+      "exact_match": false,
+      "code_bleu": 0.1561969968460128,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "out[1])",
+      "got": "out[1])",
+      "got_raw": "out[1])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "I2.data)",
+      "got": "I2.data)",
+      "got_raw": "I2.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "fvals[i]",
+      "got": "fvals[i]",
+      "got_raw": "fvals[i]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "I2.indptr)",
+      "got": "I2.indptr)",
+      "got_raw": "I2.indptr)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "exact.data)",
+      "got": "exact.data)",
+      "got_raw": "exact.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "100.0/100.0)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": false,
+      "code_bleu": 0.2686424829558855,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "225.0/100.0)",
+      "got": "100.0/100.0)",
+      "got_raw": "100.0/100.0) # 1",
+      "exact_match": false,
+      "code_bleu": 0.668740304976422,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "result.data)",
+      "got": "result.data)",
+      "got_raw": "result.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "1.2 * conv_sa",
+      "got": "conv_sa",
+      "got_raw": "conv_sa",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.7
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "1.3 * conv_sa",
+      "got": "conv_sa",
+      "got_raw": "conv_sa",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.7
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "exact.indptr)",
+      "got": "exact.indptr)",
+      "got_raw": "exact.indptr)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "expected.nnz)",
+      "got": "expected.nnz)",
+      "got_raw": "expected.nnz)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "exact.indices)",
+      "got": "exact.indices)",
+      "got_raw": "exact.indices)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "expected.data)",
+      "got": "expected.data)",
+      "got_raw": "expected.data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "residuals[-1])",
+      "got": "0.0)",
+      "got_raw": "0.0)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.1111111111111111
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "result.indptr)",
+      "got": "result.indptr)",
+      "got_raw": "result.indptr)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "result.indices)",
+      "got": "result.indices)",
+      "got_raw": "result.indices)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "rho_scale * last_rho",
+      "got": "last_rho",
+      "got_raw": "last_rho",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "rho_scale * last_rho",
+      "got": "last_rho",
+      "got_raw": "last_rho",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "Q @ coarse_candidates)",
+      "got": "coarse_candidates)",
+      "got_raw": "coarse_candidates)",
+      "exact_match": false,
+      "code_bleu": 0.3535533905932738,
+      "edit_similarity": 0.9
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "expected_Cpts.shape[0])",
+      "got": "expected_Cpts.shape[0])",
+      "got_raw": "expected_Cpts.shape[0])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "len(r2)",
+      "got": "len(r2)",
+      "got_raw": "len(r2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "len(r2)",
+      "got": "len(r2)",
+      "got_raw": "len(r2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "A.todense())",
+      "got": "A)",
+      "got_raw": "A)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "x2/norm(x2))",
+      "got": "x2/norm(x2))",
+      "got_raw": "x2/norm(x2))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "x3/norm(x3))",
+      "got": "x3/norm(x3))",
+      "got_raw": "x3/norm(x3))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "np.array([]))",
+      "got": "0.0)",
+      "got_raw": "0.0)",
+      "exact_match": false,
+      "code_bleu": 0.13432124147794275,
+      "edit_similarity": 0.23529411764705882
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "linalg.norm(A))",
+      "got": "np.linalg.norm(A))",
+      "got_raw": "np.linalg.norm(A))",
+      "exact_match": false,
+      "code_bleu": 0.7598356856515925,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "np.dot(x, P @ y))",
+      "got": "np.dot(x, P @ y))",
+      "got_raw": "np.dot(x, P @ y))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "np.linalg.norm(b)",
+      "got": "1e-10",
+      "got_raw": "1e-10",
+      "exact_match": false,
+      "code_bleu": 0.1424691910596736,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "np.array([4, 5]))",
+      "got": "[0, 1])",
+      "got_raw": "[0, 1])",
+      "exact_match": false,
+      "code_bleu": 0.18017259916438946,
+      "edit_similarity": 0.4166666666666667
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "expected.toarray())",
+      "got": "expected.toarray())",
+      "got_raw": "expected.toarray())",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "resultl2.toarray())",
+      "got": "resultl2.toarray())",
+      "got_raw": "resultl2.toarray())",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "expected.toarray())",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.6428571428571429
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "np.sqrt(h**2 + h**2))",
+      "got": "h)",
+      "got_raw": "h)",
+      "exact_match": false,
+      "code_bleu": 0.07928047433351473,
+      "edit_similarity": 0.17391304347826086
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "np.array([[3], [3]]))",
+      "got": "[3, 4, 5])",
+      "got_raw": "[3, 4, 5])",
+      "exact_match": false,
+      "code_bleu": 0.1481396818139622,
+      "edit_similarity": 0.3870967741935484
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "np.ones(u[id3].shape))",
+      "got": "1.0)",
+      "got_raw": "1.0)",
+      "exact_match": false,
+      "code_bleu": 0.08954749431862849,
+      "edit_similarity": 0.15384615384615385
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "1e-8*np.linalg.norm(b)",
+      "got": "1e-8*np.linalg.norm(b)",
+      "got_raw": "1e-8*np.linalg.norm(b)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "1e-8*precon_norm(b, ml)",
+      "got": "1e-8",
+      "got_raw": "1e-8",
+      "exact_match": false,
+      "code_bleu": 0.25226892457611433,
+      "edit_similarity": 0.2962962962962963
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "nonsymm_lvl.A.toarray())",
+      "got": "symm_lvl)",
+      "got_raw": "symm_lvl)",
+      "exact_match": false,
+      "code_bleu": 0.13432124147794275,
+      "edit_similarity": 0.5454545454545454
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "nonsymm_lvl.A.toarray())",
+      "got": "symm_lvl)",
+      "got_raw": "symm_lvl)",
+      "exact_match": false,
+      "code_bleu": 0.13432124147794275,
+      "edit_similarity": 0.5454545454545454
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "residuals2/norm(residuals2))",
+      "got": "residuals2)",
+      "got_raw": "residuals2)",
+      "exact_match": false,
+      "code_bleu": 0.20203050891044216,
+      "edit_similarity": 0.5641025641025641
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "True)",
+      "got": ")",
+      "got_raw": ")",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "True)",
+      "got": "x0.shape)",
+      "got_raw": "x0.shape)",
+      "exact_match": false,
+      "code_bleu": 0.35930411196308426,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "True)",
+      "got": "True)",
+      "got_raw": "True)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "False)",
+      "got": "False)",
+      "got_raw": "False)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[0])",
+      "got": "[0])",
+      "got_raw": "[0])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[-4])",
+      "got": "[-4])",
+      "got_raw": "[-4])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[1, 1])",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": false,
+      "code_bleu": 0.3194715521231362,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[1, 5])",
+      "got": "[0.5, 0.5])",
+      "got_raw": "[0.5, 0.5])",
+      "exact_match": false,
+      "code_bleu": 0.3303164318013807,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[4, 4])",
+      "got": "[1, 1, 1, 1, 1, 1, 1, 1])",
+      "got_raw": "[1, 1, 1, 1, 1, 1, 1, 1])",
+      "exact_match": false,
+      "code_bleu": 0.101824256461955,
+      "edit_similarity": 0.3125
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[1, 6])",
+      "got": "[1, 4])",
+      "got_raw": "[1, 4])",
+      "exact_match": false,
+      "code_bleu": 0.488923022434901,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[4, 6])",
+      "got": "[2, 5])",
+      "got_raw": "[2, 5])",
+      "exact_match": false,
+      "code_bleu": 0.3303164318013807,
+      "edit_similarity": 0.7142857142857143
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[1, 4])",
+      "got": "[1, 4])",
+      "got_raw": "[1, 4])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[72, 9])",
+      "got": "[72, 9])",
+      "got_raw": "[72, 9])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[15, 0])",
+      "got": "[15, 0])",
+      "got_raw": "[15, 0])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "(239, 1))",
+      "got": "(239, 3))",
+      "got_raw": "(239, 3))",
+      "exact_match": false,
+      "code_bleu": 0.488923022434901,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "(240, 3))",
+      "got": "(239, 3))",
+      "got_raw": "(239, 3))",
+      "exact_match": false,
+      "code_bleu": 0.6434588841607617,
+      "edit_similarity": 0.7777777777777778
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "(480, 3))",
+      "got": "(120, 4))",
+      "got_raw": "(120, 4))",
+      "exact_match": false,
+      "code_bleu": 0.3303164318013807,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[0, 0, 0])",
+      "got": "[0, 1, 2])",
+      "got_raw": "[0, 1, 2])",
+      "exact_match": false,
+      "code_bleu": 0.3688939732334406,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[0, 1, 2])",
+      "got": "[0, 1, 2])",
+      "got_raw": "[0, 1, 2])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[1, 5, 8])",
+      "got": "[0.5])",
+      "got_raw": "[0.5])",
+      "exact_match": false,
+      "code_bleu": 0.25406637407730737,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[4, 4, 1])",
+      "got": "[1, 1, 1, 1, 1, 1, 1, 1, 1])",
+      "got_raw": "[1, 1, 1, 1, 1, 1, 1, 1, 1])",
+      "exact_match": false,
+      "code_bleu": 0.18493046910349434,
+      "edit_similarity": 0.42105263157894735
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "{i-1, i+1}",
+      "got": "set(range(i))",
+      "got_raw": "set(range(i))",
+      "exact_match": false,
+      "code_bleu": 0.14447267332659466,
+      "edit_similarity": 0.08695652173913043
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "(239, 239))",
+      "got": "(10, 10))",
+      "got_raw": "(10, 10))",
+      "exact_match": false,
+      "code_bleu": 0.3303164318013807,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "(h_split.shape[0]+1)/2",
+      "got": "0.5",
+      "got_raw": "0.5",
+      "exact_match": false,
+      "code_bleu": 0.041666666666666664,
+      "edit_similarity": 0.08
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "(h_split.shape[0]-1)/2",
+      "got": "(h_split.shape[0]-1)/2",
+      "got_raw": "(h_split.shape[0]-1)/2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "(c_split.shape[0]+1)/2",
+      "got": "(c_split.shape[0]+1)/2",
+      "got_raw": "(c_split.shape[0]+1)/2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "(c_split.shape[0]-1)/2",
+      "got": "(c_split.shape[0]-1)/2",
+      "got_raw": "(c_split.shape[0]-1)/2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[0, 0, 0, 0, 1, 1, 1, 1])",
+      "got": "[0, 1, 2, 3, 4, 5, 6, 7])",
+      "got_raw": "[0, 1, 2, 3, 4, 5, 6, 7])",
+      "exact_match": false,
+      "code_bleu": 0.2861853478258715,
+      "edit_similarity": 0.52
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[1, 0, 1, 2, 1, 0, 1, 2])",
+      "got": "[0, 0, 0, 0, 0, 0, 0, 0])",
+      "got_raw": "[0, 0, 0, 0, 0, 0, 0, 0])",
+      "exact_match": false,
+      "code_bleu": 0.2239521374625909,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[0, 0, 0, 0, 1, 1, 1, 1, 2])",
+      "got": "[0, 1, 2])",
+      "got_raw": "[0, 1, 2])",
+      "exact_match": false,
+      "code_bleu": 0.3821771168817467,
+      "edit_similarity": 0.5263157894736842
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[[0, 3, 2], [0, 1, 3]])",
+      "got": "[[0, 1], [1, 2], [2, 3], [3, 0]])",
+      "got_raw": "[[0, 1], [1, 2], [2, 3], [3, 0]])",
+      "exact_match": false,
+      "code_bleu": 0.417726717020863,
+      "edit_similarity": 0.6428571428571429
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[[0., 0.], [1., 0.], [0., 1.], [1., 1.]])",
+      "got": "[[0.0, 0.0], [1.0, 0.0], [0.5, 0.8660254037844386], [0.0, 1.7320508075688772]])",
+      "got_raw": "[[0.0, 0.0], [1.0, 0.0], [0.5, 0.8660254037844386], [0.0, 1.7320508075688772]])",
+      "exact_match": false,
+      "code_bleu": 0.2179258811646635,
+      "edit_similarity": 0.65
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[[0, 4, 3], [1, 5, 4], [0, 1, 4], [1, 2, 5]])",
+      "got": "[[0, 1], [1, 2], [2, 0]])",
+      "got_raw": "[[0, 1], [1, 2], [2, 0]])",
+      "exact_match": false,
+      "code_bleu": 0.3499857449901785,
+      "edit_similarity": 0.5428571428571428
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "[[0., 0.], [0.5, 0.], [1., 0.], [0., 1.], [0.5, 1.], [1., 1.]])",
+      "got": "[[0.0, 0.0], [1.0, 0.0], [0.5, 0.8660254037844386], [0.0, 1.7320508075688772], [1.0, 1.7320508075688772]])",
+      "got_raw": "[[0.0, 0.0], [1.0, 0.0], [0.5, 0.8660254037844386], [0.0, 1.7320508075688772], [1.0, 1.7320508075688772]])",
+      "exact_match": false,
+      "code_bleu": 0.2534937725951494,
+      "edit_similarity": 0.5325443786982249
+    },
+    {
+      "repo": "pyamg/pyamg",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "9",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "torch_frame.stype('numerical')",
+      "got": "[0",
+      "got_raw": "[0, 1, 2, 3, 4, 5, 6, 7, 8]",
+      "exact_match": false,
+      "code_bleu": 0.15060033393497413,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "torch_frame.stype('embedding')",
+      "got": "torch_frame.stype('embedding')",
+      "got_raw": "torch_frame.stype('embedding')",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "torch_frame.stype('categorical')",
+      "got": "torch_frame.stype('categorical')",
+      "got_raw": "torch_frame.stype('categorical')",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "torch_frame.stype('text_embedded')",
+      "got": "torch_frame.stype('text_embedded')",
+      "got_raw": "torch_frame.stype('text_embedded')",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "[]",
+      "got": "['target']",
+      "got_raw": "['target']",
+      "exact_match": false,
+      "code_bleu": 0.49999999999999994,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "[4, 3, 1]",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.07142857142857142,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "task_type",
+      "got": "task_type",
+      "got_raw": "task_type",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "num_classes",
+      "got": "num_classes",
+      "got_raw": "num_classes",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "len(dataset)",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "([1, 0], [3, 1])",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": false,
+      "code_bleu": 0.038461538461538464,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "([0, 1], [1, 3])",
+      "got": "([1, 1, 1, 1], [4, 0])",
+      "got_raw": "([1, 1, 1, 1], [4, 0])",
+      "exact_match": false,
+      "code_bleu": 0.39317519708167065,
+      "edit_similarity": 0.6842105263157895
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "['cat_1', 'cat_2']",
+      "got": "_frame.col_types_dict[torch_frame.numerical]",
+      "got_raw": "_frame.col_types_dict[torch_frame.numerical]",
+      "exact_match": false,
+      "code_bleu": 0.17747405280050269,
+      "edit_similarity": 0.12903225806451613
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "test_dataset.col_stats",
+      "got": "test_dataset.col_stats",
+      "got_raw": "test_dataset.col_stats",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "['num_1', 'num_2', 'num_3']",
+      "got": "ortframe()",
+      "got_raw": "ortframe()",
+      "exact_match": false,
+      "code_bleu": 0.16282193263962697,
+      "edit_similarity": 0.05405405405405406
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "4",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "''",
+      "got": "''",
+      "got_raw": "''",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "'B'",
+      "got": "'B'",
+      "got_raw": "'B'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "ser)",
+      "got": "ser)",
+      "got_raw": "ser)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(2, 7)",
+      "got": "(2,)",
+      "got_raw": "(2,)",
+      "exact_match": false,
+      "code_bleu": 0.565685424949238,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "torch.long",
+      "got": "torch.int64",
+      "got_raw": "torch.int64",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "torch.float",
+      "got": "torch.float",
+      "got_raw": "torch.float",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "torch.float32",
+      "got": "torch.float32",
+      "got_raw": "torch.float32",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(num_sentences + 1, 1, -1)",
+      "got": "(num_sentences, out_channels)",
+      "got_raw": "(num_sentences, out_channels)",
+      "exact_match": false,
+      "code_bleu": 0.2041241452319315,
+      "edit_similarity": 0.6181818181818182
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(num_sentences + 1, out_channels)",
+      "got": "(num_sentences + 1, 1, out_channels)",
+      "got_raw": "(num_sentences + 1, 1, out_channels)",
+      "exact_match": false,
+      "code_bleu": 0.7186082239261685,
+      "edit_similarity": 0.9565217391304348
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "pd.Series(['A', 'B', None, None, 'B']))",
+      "got": "ser)",
+      "got_raw": "ser)",
+      "exact_match": false,
+      "code_bleu": 0.06320999598962011,
+      "edit_similarity": 0.13953488372093023
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "pd.Series([[0.1, 0.5], [0.3], None, [0.2, np.nan], None, None]))",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "-1",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "met)",
+      "got": "met)",
+      "got_raw": "met)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "12345.",
+      "got": "12345.",
+      "got_raw": "12345.",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "met[:])",
+      "got": "met)",
+      "got_raw": "met)",
+      "exact_match": false,
+      "code_bleu": 0.23784142300054423,
+      "edit_similarity": 0.7272727272727273
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "num_rows",
+      "got": "num_rows",
+      "got_raw": "num_rows",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "num_cols",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "met[:3])",
+      "got": "met[:3])",
+      "got_raw": "met[:3])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "met[3:])",
+      "got": "met[3:])",
+      "got_raw": "met[3:])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "1",
+      "got": "num_cols",
+      "got_raw": "num_cols",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "multi_nested_tensor.shape[0]",
+      "got": "len(multi_nested_tensor)",
+      "got_raw": "len(multi_nested_tensor)",
+      "exact_match": false,
+      "code_bleu": 0.23953607464205617,
+      "edit_similarity": 0.7307692307692307
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "multi_nested_tensor.shape[1]",
+      "got": "multi_nested_tensor.shape[1]",
+      "got_raw": "multi_nested_tensor.shape[1]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "RuntimeError, match=\"The length of each row must be the same\")",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "year_range[1]",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.14285714285714285
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "expected_col_stats",
+      "got": "expected_col_stats",
+      "got_raw": "expected_col_stats",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "TypeError, match='Numerical series contains invalid entries.')",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.1917808219178082
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "{ StatType.EMB_DIM: emb_dim, }",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.125
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "{ StatType.COUNT: (['a', 'b', 'c'], [3, 1, 1]), }",
+      "got": "{0: {'n': 5, 'unique': 3, 'freq': [0.2, 0.4, 0.4]}}",
+      "got_raw": "{0: {'n': 5, 'unique': 3, 'freq': [0.2, 0.4, 0.4]}}",
+      "exact_match": false,
+      "code_bleu": 0.09235852770807895,
+      "edit_similarity": 0.36
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "{ StatType.MULTI_COUNT: (['a', 'c', 'b'], [4, 3, 2]), }",
+      "got": "{0: {'n': 5, 'unique': 3, 'nunique': 2, 'top': 'a', 'freq': 2}}",
+      "got_raw": "{0: {'n': 5, 'unique': 3, 'nunique': 2, 'top': 'a', 'freq': 2}}",
+      "exact_match": false,
+      "code_bleu": 0.09792991473477164,
+      "edit_similarity": 0.22033898305084745
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "{ StatType.MEAN: 2.0, StatType.STD: 0.816496580927726, StatType.QUANTILES: [1.0, 1.5, 2.0, 2.5, 3.0], }",
+      "got": "{StatType.MEAN: 2.0}",
+      "got_raw": "{StatType.MEAN: 2.0}",
+      "exact_match": false,
+      "code_bleu": 0.202869740240996,
+      "edit_similarity": 0.3252032520325203
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "{ StatType.MEAN: 3.5, StatType.STD: 1.707825127659933, StatType.QUANTILES: [1.0, 2.25, 3.5, 4.75, 6.0], }",
+      "got": "{StatType.MEAN: 2.5, StatType.STD: 1.5}",
+      "got_raw": "{StatType.MEAN: 2.5, StatType.STD: 1.5}",
+      "exact_match": false,
+      "code_bleu": 0.2729189663370838,
+      "edit_similarity": 0.5277777777777778
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "{ StatType.MEAN: np.nan, StatType.STD: np.nan, StatType.QUANTILES: [np.nan, np.nan, np.nan, np.nan, np.nan], }",
+      "got": "{StatType.MEAN: np.nan}",
+      "got_raw": "{StatType.MEAN: np.nan}",
+      "exact_match": false,
+      "code_bleu": 0.18482768255655918,
+      "edit_similarity": 0.3458646616541353
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "16",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "tf2",
+      "got": "tf2",
+      "got_raw": "tf2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "tf1",
+      "got": "tf1",
+      "got_raw": "tf1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "tf)",
+      "got": "tf",
+      "got_raw": "tf, False)",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "0",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "8124",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "('Dota2', {})",
+      "got": "('TabularBenchmark', {'name': 'house_16H'})",
+      "got_raw": "('TabularBenchmark', {'name': 'house_16H'})",
+      "exact_match": false,
+      "code_bleu": 0.24446151121745052,
+      "edit_similarity": 0.32142857142857145
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "('Mushroom', {})",
+      "got": "('AdultCensusIncome', {'scale': 'medium'})",
+      "got_raw": "('AdultCensusIncome', {'scale': 'medium'})",
+      "exact_match": false,
+      "code_bleu": 0.24446151121745052,
+      "edit_similarity": 0.41379310344827586
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "('PokerHand', {})",
+      "got": "('TabularBenchmark', {'name': 'Diabetes130US'})",
+      "got_raw": "('TabularBenchmark', {'name': 'Diabetes130US'})",
+      "exact_match": false,
+      "code_bleu": 0.24446151121745052,
+      "edit_similarity": 0.3125
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "('BankMarketing', {})",
+      "got": "('Pulsar', {})",
+      "got_raw": "('Pulsar', {})",
+      "exact_match": false,
+      "code_bleu": 0.6434588841607617,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "\"DataFrameBenchmark(\\n\" \" task_type=binary_classification,\\n\" \" scale=small,\\n\" \" idx=1,\\n\" \" cls=Mushroom()\\n\" \")\"",
+      "got": "f\"DataFrameBenchmark(task_type={TaskType.BINARY_CLASSIFICATION}, scale={1}, idx=1)\")",
+      "got_raw": "f\"DataFrameBenchmark(task_type={TaskType.BINARY_CLASSIFICATION}, scale={1}, idx=1)\")",
+      "exact_match": false,
+      "code_bleu": 0.18995892141289814,
+      "edit_similarity": 0.49246231155778897
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "0",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "6364",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "5091",
+      "got": "6364",
+      "got_raw": "6364",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "1273",
+      "got": "2273",
+      "got_raw": "2273",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "('Mercari', {})",
+      "got": "('AmazonReview', {})",
+      "got_raw": "('AmazonReview', {})",
+      "exact_match": false,
+      "code_bleu": 0.6434588841607617,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "('AmazonFineFoodReviews', {})",
+      "got": "('HuggingFaceDatasetDict', {",
+      "got_raw": "('HuggingFaceDatasetDict', {\n                'path': 'maharshipandya/spotify-tracks-dataset',\n                'columns': [\n                    'artists', 'album_name', 'track_name', 'popularity',\n                    'duration_ms', 'explicit', 'danceability', 'energy', 'key',\n                    'loudness', 'mode', 'speechiness', 'acousticness',\n                    'instrumentalness', 'liveness', 'valence', 'tempo',\n                    'time_signature', 'track_genre'\n                ],\n                'target_col': 'track_genre',\n            })",
+      "exact_match": false,
+      "code_bleu": 0.3387551654364098,
+      "edit_similarity": 0.38596491228070173
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "('MultimodalTextBenchmark', { 'name': 'news_channel' })",
+      "got": "('MultimodalTextBenchmark', {",
+      "got_raw": "('MultimodalTextBenchmark', {\n                'name': 'product_sentiment_machine_hack2'\n            })",
+      "exact_match": false,
+      "code_bleu": 0.4444444444444444,
+      "edit_similarity": 0.6904761904761905
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "('MultimodalTextBenchmark', { 'name': 'wine_reviews' })",
+      "got": "('MultimodalTextBenchmark', {",
+      "got_raw": "('MultimodalTextBenchmark', {\n                'name': 'fake_job_postings100K'\n            })",
+      "exact_match": false,
+      "code_bleu": 0.4444444444444444,
+      "edit_similarity": 0.6904761904761905
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "\"DataFrameTextBenchmark(\\n\" \" task_type=multiclass_classification,\\n\" \" scale=small,\\n\" \" idx=0,\\n\" \" cls=MultimodalTextBenchmark()\\n\" \")\"",
+      "got": "f\"TextBenchmarkDataset(root={temp_dir}, task_type=\"",
+      "got_raw": "f\"TextBenchmarkDataset(root={temp_dir}, task_type=\"",
+      "exact_match": false,
+      "code_bleu": 0.15060033393497413,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "1",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "num_rows",
+      "got": "num_rows",
+      "got_raw": "num_rows",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "4",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "10",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "1000209",
+      "got": "1000",
+      "got_raw": "1000",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7272727272727273
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "torch.int64",
+      "got": "torch.int64",
+      "got_raw": "torch.int64",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "2",
+      "got": "6",
+      "got_raw": "6",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "8",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "891",
+      "got": "891",
+      "got_raw": "891",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(891, )",
+      "got": "(891,)",
+      "got_raw": "(891,)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 0.9230769230769231
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(891, 4)",
+      "got": "(891, 5)",
+      "got_raw": "(891, 5)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.875
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "0",
+      "got": "0.0)",
+      "got_raw": "0.0)",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "score",
+      "got": "score",
+      "got_raw": "score",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "metric",
+      "got": "loaded_gbdt.metric",
+      "got_raw": "loaded_gbdt.metric",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(10, 8)",
+      "got": "(10, 8)",
+      "got_raw": "(10, 8)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(10, 4, 8)",
+      "got": "(10, 4, 8)",
+      "got_raw": "(10, 4, 8)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(10, 4, 16)",
+      "got": "(10, 16, 16)",
+      "got_raw": "(10, 16, 16)",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8695652173913043
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "['num1', 'num2', 'cat1', 'cat2']",
+      "got": "['num1', 'num2', 'cat1', 'cat2']",
+      "got_raw": "['num1', 'num2', 'cat1', 'cat2']",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(batch_size, num_cols, channels)",
+      "got": "(x.shape[0], x.shape[1], num_heads, channels // num_heads)",
+      "got_raw": "(x.shape[0], x.shape[1], num_heads, channels // num_heads)",
+      "exact_match": false,
+      "code_bleu": 0.0866852806734874,
+      "edit_similarity": 0.4666666666666667
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(10, 8)",
+      "got": "(10, 1, 8)",
+      "got_raw": "(10, 1, 8)",
+      "exact_match": false,
+      "code_bleu": 0.48109772909788073,
+      "edit_similarity": 0.8235294117647058
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(10, 3, 8)",
+      "got": "(10, 9, 8)",
+      "got_raw": "(10, 9, 8)",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(batch_size, num_cols, channels)",
+      "got": "(x.shape[0], x.shape[1], num_heads, channels)",
+      "got_raw": "(x.shape[0], x.shape[1], num_heads, channels)",
+      "exact_match": false,
+      "code_bleu": 0.1326475916741222,
+      "edit_similarity": 0.5454545454545454
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(batch_size, num_prompts, channels)",
+      "got": "(x.shape)",
+      "got_raw": "(x.shape)",
+      "exact_match": false,
+      "code_bleu": 0.2158125283826263,
+      "edit_similarity": 0.2727272727272727
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(batch_size, out_channels)",
+      "got": "(x.shape)",
+      "got_raw": "(x.shape)",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.34285714285714286
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(batch_size, out_channels)",
+      "got": "(x_prompt.shape)",
+      "got_raw": "(x_prompt.shape)",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.38095238095238093
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "len(dataset.df.dropna())",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.05,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(feat_cat.size(0), feat_cat.size(1), 8)",
+      "got": "(10, 8)",
+      "got_raw": "(10, 8)",
+      "exact_match": false,
+      "code_bleu": 0.1580249899740503,
+      "edit_similarity": 0.2608695652173913
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(feat_num.size(0), feat_num.size(1), 8)",
+      "got": "(10, 8)",
+      "got_raw": "(10, 8)",
+      "exact_match": false,
+      "code_bleu": 0.1580249899740503,
+      "edit_similarity": 0.2608695652173913
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(feat_multicat.size(0), feat_multicat.size(1), 8)",
+      "got": "feat_multicat.shape",
+      "got_raw": "feat_multicat.shape",
+      "exact_match": false,
+      "code_bleu": 0.10493003955906363,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(feat_timestamp.size(0), feat_timestamp.size(1), 8)",
+      "got": "(10, 8)",
+      "got_raw": "(10, 8)",
+      "exact_match": false,
+      "code_bleu": 0.1580249899740503,
+      "edit_similarity": 0.20689655172413793
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "tensor_frame.feat_dict[ stype.text_tokenized].keys()",
+      "got": "tensor_frame.feat_dict[stype.text_tokenized].keys()",
+      "got_raw": "tensor_frame.feat_dict[stype.text_tokenized].keys()",
+      "exact_match": false,
+      "code_bleu": 1.0,
+      "edit_similarity": 0.9902912621359223
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "( num_rows, len(tensor_frame.col_names_dict[data_stype]), out_channels, )",
+      "got": "(len(col_names), out_channels)",
+      "got_raw": "(len(col_names), out_channels)",
+      "exact_match": false,
+      "code_bleu": 0.1907082808182838,
+      "edit_similarity": 0.5825242718446602
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "( num_rows, len(tensor_frame.col_names_dict[stype.embedding]), out_channels, )",
+      "got": "(num_rows, out_channels)",
+      "got_raw": "(num_rows, out_channels)",
+      "exact_match": false,
+      "code_bleu": 0.16785418874204266,
+      "edit_similarity": 0.47058823529411764
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "( num_rows, len(tensor_frame.col_names_dict[stype.text_tokenized]), out_channels, )",
+      "got": "(num_rows, out_channels)",
+      "got_raw": "(num_rows, out_channels)",
+      "exact_match": false,
+      "code_bleu": 0.16785418874204266,
+      "edit_similarity": 0.4485981308411215
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "tensor_frame",
+      "got": "tensor_frame",
+      "got_raw": "tensor_frame",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "ValueError, match=\"is an invalid stype\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.43137254901960786
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(num_rows, tensor_frame.num_cols, out_channels)",
+      "got": "(num_rows, out_channels)",
+      "got_raw": "(num_rows, out_channels)",
+      "exact_match": false,
+      "code_bleu": 0.39283710065919303,
+      "edit_similarity": 0.676056338028169
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "[ \"num_1\", \"num_2\", \"num_3\", \"cat_1\", \"cat_2\", \"text_tokenized_1\", \"text_tokenized_2\", \"multicat_1\", \"multicat_2\", \"multicat_3\", \"multicat_4\", \"timestamp_0\", \"timestamp_1\", \"timestamp_2\", \"emb_1\", \"emb_2\", \"text_embedded_1\", \"text_embedded_2\", ]",
+      "got": "tensor_frame.col_names",
+      "got_raw": "tensor_frame.col_names",
+      "exact_match": false,
+      "code_bleu": 0.029993513907299704,
+      "edit_similarity": 0.0299625468164794
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "input_tensor.shape + (out_size, )",
+      "got": "size",
+      "got_raw": "size",
+      "exact_match": false,
+      "code_bleu": 0.0625,
+      "edit_similarity": 0.21621621621621623
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "input_tensor.shape + (out_size, )",
+      "got": "size",
+      "got_raw": "size",
+      "exact_match": false,
+      "code_bleu": 0.0625,
+      "edit_similarity": 0.21621621621621623
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "expected_graph_breaks",
+      "got": "expected_graph_breaks",
+      "got_raw": "expected_graph_breaks",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "tensor_frame.y.shape",
+      "got": "()",
+      "got_raw": "()",
+      "exact_match": false,
+      "code_bleu": 0.18072040072196896,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(batch_size, out_channels)",
+      "got": "(batch_size, out_channels)",
+      "got_raw": "(batch_size, out_channels)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(batch_size, out_channels)",
+      "got": "(batch_size, out_channels)",
+      "got_raw": "(batch_size, out_channels)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(batch_size, out_channels)",
+      "got": "(batch_size, out_channels)",
+      "got_raw": "(batch_size, out_channels)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(batch_size, out_channels)",
+      "got": "(batch_size, out_channels)",
+      "got_raw": "(batch_size, out_channels)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(batch_size, out_channels)",
+      "got": "(batch_size, out_channels)",
+      "got_raw": "(batch_size, out_channels)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(len(tensor_frame), out_channels)",
+      "got": "(batch_size, out_channels)",
+      "got_raw": "(batch_size, out_channels)",
+      "exact_match": false,
+      "code_bleu": 0.3358031036948569,
+      "edit_similarity": 0.6101694915254238
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "(batch_size, num_layers, out_channels)",
+      "got": "(batch_size, out_channels)",
+      "got_raw": "(batch_size, out_channels)",
+      "exact_match": false,
+      "code_bleu": 0.5050762722761053,
+      "edit_similarity": 0.8125
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "col_stats",
+      "got": "col_stats",
+      "got_raw": "col_stats",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "total_cols",
+      "got": "total_numerical_cols)",
+      "got_raw": "total_numerical_cols)",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.6451612903225806
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "out.feat_dict",
+      "got": "out.feat_cols",
+      "got_raw": "out.feat_cols",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.7692307692307693
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "out.col_names_dict",
+      "got": "out.feat_cols",
+      "got_raw": "out.feat_cols",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.5161290322580645
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "out.feat_dict.keys()",
+      "got": "loaded_out.col_names_dict.keys())",
+      "got_raw": "loaded_out.col_names_dict.keys())",
+      "exact_match": false,
+      "code_bleu": 0.46713797772820004,
+      "edit_similarity": 0.6415094339622641
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "len(categorical_features)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": false,
+      "code_bleu": 0.2686424829558855,
+      "edit_similarity": 0.07407407407407407
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "out.col_names_dict.keys()",
+      "got": "loaded_out.col_types",
+      "got_raw": "loaded_out.col_types",
+      "exact_match": false,
+      "code_bleu": 0.19362900077353815,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "loaded_out.col_names_dict",
+      "got": "loaded_out.col_names_dict)",
+      "got_raw": "loaded_out.col_names_dict)",
+      "exact_match": false,
+      "code_bleu": 0.668740304976422,
+      "edit_similarity": 0.9803921568627451
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "transformed_col_stats[col]",
+      "got": "col",
+      "got_raw": "col",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.20689655172413793
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "'num_1'",
+      "got": "'num_1')",
+      "got_raw": "'num_1')",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.9333333333333333
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "tensor_frame.feat_dict[ stype.numerical].size()",
+      "got": "tensor_frame.feat_dict[stype.numerical].size())",
+      "got_raw": "tensor_frame.feat_dict[stype.numerical].size())",
+      "exact_match": false,
+      "code_bleu": 0.9193227152249185,
+      "edit_similarity": 0.9787234042553191
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "set( tensor_frame.col_names_dict[stype.numerical])",
+      "got": "set(dataset.col_names))",
+      "got_raw": "set(dataset.col_names))",
+      "exact_match": false,
+      "code_bleu": 0.17675757489165944,
+      "edit_similarity": 0.4657534246575342
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "tf",
+      "got": "tf",
+      "got_raw": "tf",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "num_rows * num_repeats",
+      "got": "num_rows * num_repeats",
+      "got_raw": "num_rows * num_repeats",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "dataset.col_to_stype",
+      "got": "{",
+      "got_raw": "{ \"text\": torch_frame.text_embedded, \"text2\": torch_frame.text_embedded, \"text3\": torch_frame.text_embedded, \"text4\": torch_frame.text_embedded, \"text5\": torch_frame.text_embedded, \"text6\": torch_frame.text_embedded, \"text7\": torch_frame.text_embedded, \"text8\": torch_frame.text_embedded, \"text9\": torch_frame.text_embedded, \"text10\": torch_frame.text_embedded, \"text11\": torch_frame.text_embedded, \"text12\": torch_frame.text_embedded, \"text13\":",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "torch_frame.categorical",
+      "got": "torch_frame.bool",
+      "got_raw": "torch_frame.bool",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.717948717948718
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "torch_frame.multicategorical",
+      "got": "torch_frame.multicategorical",
+      "got_raw": "torch_frame.multicategorical",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "tf",
+      "got": "tf",
+      "got_raw": "tf",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "col_stats",
+      "got": "col_stats",
+      "got_raw": "col_stats",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "new_dataset.col_stats",
+      "got": "new_dataset.col_stats",
+      "got_raw": "new_dataset.col_stats",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "new_dataset.tensor_frame",
+      "got": "new_dataset.tensor_frame",
+      "got_raw": "new_dataset.tensor_frame",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "4 * 8 * 4",
+      "got": "32",
+      "got_raw": "32",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "12 * 4 + 7 * 8",
+      "got": "12 * 4",
+      "got_raw": "12 * 4",
+      "exact_match": false,
+      "code_bleu": 0.36038417796587763,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "2 * 10 * 4 + 4 * 8",
+      "got": "2 * 10 * 4",
+      "got_raw": "2 * 10 * 4",
+      "exact_match": false,
+      "code_bleu": 0.5555555555555556,
+      "edit_similarity": 0.7142857142857143
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "2 * num_bytes(data)",
+      "got": "2 * num_bytes(data)",
+      "got_raw": "2 * num_bytes(data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "3 * num_bytes(data)",
+      "got": "3 * num_bytes(data)",
+      "got_raw": "3 * num_bytes(data)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "int(num_data * val_ratio)",
+      "got": "int(num_data * val_ratio)",
+      "got_raw": "int(num_data * val_ratio)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "int(num_data * test_ratio)",
+      "got": "int(num_data * test_ratio)",
+      "got_raw": "int(num_data * test_ratio)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pyg-team/pytorch-frame",
+      "expected": "int(num_data * train_ratio)",
+      "got": "16",
+      "got_raw": "16",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "\"\"",
+      "got": "ref_metadata.package_version",
+      "got_raw": "ref_metadata.package_version",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "ref_metadata_replaced",
+      "got": "ref_metadata_replaced",
+      "got_raw": "ref_metadata_replaced",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.err",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "expected_string",
+      "got": "expected_string",
+      "got_raw": "expected_string",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "expected_string[:chars_to_test]",
+      "got": "expected_string",
+      "got_raw": "expected_string",
+      "exact_match": false,
+      "code_bleu": 0.11892071150027211,
+      "edit_similarity": 0.6521739130434783
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "exposed",
+      "got": "exposed",
+      "got_raw": "exposed",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.out",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "SystemExit)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.2727272727272727
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.out",
+      "got": "captured.out",
+      "got_raw": "captured.out",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.err",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "str(env_dir)",
+      "got": "env_dir",
+      "got_raw": "env_dir",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.7368421052631579
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "home / \"test\"",
+      "got": "home / \"test\"",
+      "got_raw": "home / \"test\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "{pkg_name}",
+      "got": "{pkg_name}",
+      "got_raw": "{pkg_name}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "caplog.text",
+      "got": "caplog.text",
+      "got_raw": "caplog.text",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "capsys.readouterr().err",
+      "got": "capsys.readouterr().err",
+      "got_raw": "capsys.readouterr().err",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "{pkg for pkg, _ in packages}",
+      "got": "set(pkg for pkg, _ in packages)",
+      "got_raw": "set(pkg for pkg, _ in packages)",
+      "exact_match": false,
+      "code_bleu": 0.6786502681586726,
+      "edit_similarity": 0.8813559322033898
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "caplog.text",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.out",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.out",
+      "got": "captured.out",
+      "got_raw": "captured.out",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.err",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "log_contents",
+      "got": "log_contents",
+      "got_raw": "log_contents",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "\"py\"",
+      "got": "\"py\"",
+      "got_raw": "\"py\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "str(e)",
+      "got": "str(e.value)",
+      "got_raw": "str(e.value)",
+      "exact_match": false,
+      "code_bleu": 0.4347208719449915,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "\"python\"",
+      "got": "\"python\"",
+      "got_raw": "\"python\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "PipxError)",
+      "got": "PipxError)",
+      "got_raw": "PipxError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "python_path",
+      "got": "python_path",
+      "got_raw": "python_path",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "\"WindowsApps\"",
+      "got": "\"3.8\"",
+      "got_raw": "\"3.8\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "sys.executable",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.1111111111111111
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "InterpreterResolutionError)",
+      "got": "PipxError)",
+      "got_raw": "PipxError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.3783783783783784
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "{}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "[\"black\"]",
+      "got": "[\"black\"]",
+      "got_raw": "[\"black\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.err",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.out",
+      "got": "captured.out",
+      "got_raw": "captured.out",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "black_package_ref)",
+      "got": "black_package_ref)",
+      "got_raw": "black_package_ref)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "0)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.out",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.out.strip()",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": false,
+      "code_bleu": 0.2548300960720116,
+      "edit_similarity": 0.625
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "ValueError, match=\"raised in test to exit early\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.36666666666666664
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "caplog.text",
+      "got": "caplog.text",
+      "got_raw": "caplog.text",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "package_name_out",
+      "got": "package_name_out",
+      "got_raw": "package_name_out",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "package_spec_out",
+      "got": "package_spec_out",
+      "got_raw": "package_spec_out",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "package_or_url_correct",
+      "got": "package_or_url_correct",
+      "got_raw": "package_or_url_correct",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "PipxError, match=r\"^Unable to parse package spec\")",
+      "got": "PipxError)",
+      "got_raw": "PipxError)",
+      "exact_match": false,
+      "code_bleu": 0.16988673071467442,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "caplog.text",
+      "got": "caplog.text",
+      "got_raw": "caplog.text",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.err",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.out",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "{}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "{\"black\"}",
+      "got": "{PKG[\"black\"][\"spec\"]}",
+      "got_raw": "{PKG[\"black\"][\"spec\"]}",
+      "exact_match": false,
+      "code_bleu": 0.16784459625186196,
+      "edit_similarity": 0.5806451612903226
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "PipxError)",
+      "got": "PipxError)",
+      "got_raw": "PipxError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "black_package_ref)",
+      "got": "black_package_ref)",
+      "got_raw": "black_package_ref)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "pycowsay_package_ref)",
+      "got": "PYCOWSAY)",
+      "got_raw": "PYCOWSAY)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.06666666666666667
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "getattr(pipx_metadata2, attribute)",
+      "got": "getattr(pipx_metadata2, attribute)",
+      "got_raw": "getattr(pipx_metadata2, attribute)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.out",
+      "got": "captured.out",
+      "got_raw": "captured.out",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.err",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "capsys.readouterr().out",
+      "got": "capsys.readouterr().str",
+      "got_raw": "capsys.readouterr().str",
+      "exact_match": false,
+      "code_bleu": 0.8408964152537145,
+      "edit_similarity": 0.9130434782608695
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.err.replace(\"\\n\", \" \")",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": false,
+      "code_bleu": 0.25226892457611433,
+      "edit_similarity": 0.5581395348837209
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "caplog.text",
+      "got": "caplog.text",
+      "got_raw": "caplog.text",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.out",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "\"2\"",
+      "got": "\"2\"",
+      "got_raw": "\"2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "test_str",
+      "got": "test_str",
+      "got_raw": "test_str",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "SystemExit",
+      "got": "SystemExit",
+      "got_raw": "SystemExit",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "caplog.text",
+      "got": "caplog.text",
+      "got_raw": "caplog.text",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "SystemExit)",
+      "got": "RuntimeError)",
+      "got_raw": "RuntimeError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "needs_upgrade",
+      "got": "needs_upgrade",
+      "got_raw": "needs_upgrade",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.out",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "caplog.text",
+      "got": "caplog.text",
+      "got_raw": "caplog.text",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.out",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "caplog.text",
+      "got": "caplog.text",
+      "got_raw": "caplog.text",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.out",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.err",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.err",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.out",
+      "got": "captured.out",
+      "got_raw": "captured.out",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "caplog.text",
+      "got": "caplog.text",
+      "got_raw": "caplog.text",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "captured.out",
+      "got": "captured.err",
+      "got_raw": "captured.err",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "pypa/pipx",
+      "expected": "False",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "help_text",
+      "got": "help_text",
+      "got_raw": "help_text",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "''",
+      "got": "'",
+      "got_raw": "'  say [-h] text'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "'tag'",
+      "got": "'argparse-app'",
+      "got_raw": "'argparse-app'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.42105263157894735
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "'True'",
+      "got": "'we'",
+      "got_raw": "'we'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "out[1]",
+      "got": "err",
+      "got_raw": "err",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "\"foo\\n\"",
+      "got": "\"foo\\n\"",
+      "got_raw": "\"foo\\n\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "'custom'",
+      "got": "'foo'",
+      "got_raw": "'foo'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.46153846153846156
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "['10.0']",
+      "got": "\"foo",
+      "got_raw": "\"foo 2 5.0\\n\"",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "['hello']",
+      "got": "'Hello",
+      "got_raw": "'Hello, world!\\n'",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "TypeError)",
+      "got": "TypeError)",
+      "got_raw": "TypeError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "out2",
+      "got": "out2",
+      "got_raw": "out2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "show_table",
+      "got": "show_table",
+      "got_raw": "show_table",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "\"--myflag\"",
+      "got": "'m'",
+      "got_raw": "'m'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.15384615384615385
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "num_aliases",
+      "got": "num_aliases",
+      "got_raw": "num_aliases",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "display_meta",
+      "got": "display_meta",
+      "got_raw": "display_meta",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "CustomCompleter",
+      "got": "CustomCompleter",
+      "got_raw": "CustomCompleter",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "completions[0].text",
+      "got": "completions[0]",
+      "got_raw": "completions[0]",
+      "exact_match": false,
+      "code_bleu": 0.6666666666666666,
+      "edit_similarity": 0.8484848484848485
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "\"Hint:\\n  no_help_pos\"",
+      "got": "'foo'",
+      "got_raw": "'foo'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.14814814814814814
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "completions.completion_error",
+      "got": "completions",
+      "got_raw": "completions",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.5641025641025641
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "len(ac_app.num_completion_items)",
+      "got": "len(ac_app.num_completion_item_choices)",
+      "got_raw": "len(ac_app.num_completion_item_choices)",
+      "exact_match": false,
+      "code_bleu": 0.6434588841607617,
+      "edit_similarity": 0.9014084507042254
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "expected_completions.to_strings()",
+      "got": "expected_completions.to_strings()",
+      "got_raw": "expected_completions.to_strings()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "out",
+      "got": "out",
+      "got_raw": "out",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "err",
+      "got": "err",
+      "got_raw": "err",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "'1'",
+      "got": "\"1\"",
+      "got_raw": "\"1\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "'2'",
+      "got": "'2'",
+      "got_raw": "'2'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "out[0]",
+      "got": "out",
+      "got_raw": "out",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "out[1]",
+      "got": "out[1]",
+      "got_raw": "out[1]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "err[2]",
+      "got": "err",
+      "got_raw": "err",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "(0, 3)",
+      "got": "(0, 3)",
+      "got_raw": "(0, 3)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "(1, 5)",
+      "got": "(1, 5)",
+      "got_raw": "(1, 5)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "(2, 5)",
+      "got": "(2, 5)",
+      "got_raw": "(2, 5)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "new_val",
+      "got": "new_val",
+      "got_raw": "new_val",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "TypeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "''",
+      "got": "''",
+      "got_raw": "''",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "['10.0']",
+      "got": "'10\\n'",
+      "got_raw": "'10\\n'",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "['((baz))']",
+      "got": "'baz\\n'",
+      "got_raw": "'baz\\n'",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.5555555555555556
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "'Base command help'",
+      "got": "''",
+      "got_raw": "''",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.19047619047619047
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "'Default Category'",
+      "got": "\"Base\"",
+      "got_raw": "\"Base\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "3",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "2",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "3",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "out",
+      "got": "out",
+      "got_raw": "out",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "err",
+      "got": "err",
+      "got_raw": "err",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "app",
+      "got": "app",
+      "got_raw": "app",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "matches",
+      "got": "matches",
+      "got_raw": "matches",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "'\"'",
+      "got": "'\"'",
+      "got_raw": "'\"'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "\"'\"",
+      "got": "\"'",
+      "got_raw": "\"' \"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "items",
+      "got": "items",
+      "got_raw": "items",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "tokens",
+      "got": "tokens",
+      "got_raw": "tokens",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "values",
+      "got": "values",
+      "got_raw": "values",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "display",
+      "got": "\"Apple\"",
+      "got_raw": "\"Apple\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "\"Apple\"",
+      "got": "\"apple\"",
+      "got_raw": "\"apple\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "all_nums",
+      "got": "all_nums",
+      "got_raw": "all_nums",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "raw_tokens",
+      "got": "raw_tokens",
+      "got_raw": "raw_tokens",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "items[::-1]",
+      "got": "items[::-1]",
+      "got_raw": "items[::-1]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "completions",
+      "got": "completions",
+      "got_raw": "completions",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "7",
+      "got": "7",
+      "got_raw": "7",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "CompleteStyle.COLUMN",
+      "got": "CompleteStyle.BASIC",
+      "got_raw": "CompleteStyle.BASIC",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.7692307692307693
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "CompleteStyle.MULTI_COLUMN",
+      "got": "CompleteStyle.MULTI_COLUMN",
+      "got_raw": "CompleteStyle.MULTI_COLUMN",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "3",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "5",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "1",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "''",
+      "got": "Statement()",
+      "got_raw": "Statement()",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "1",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "styled",
+      "got": "styled",
+      "got_raw": "styled",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "\"arg\")",
+      "got": "[\"arg\"])",
+      "got_raw": "[\"arg\"])",
+      "exact_match": false,
+      "code_bleu": 0.3976353643835253,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "unstyled",
+      "got": "unstyled",
+      "got_raw": "unstyled",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "bar_text",
+      "got": "foo_text",
+      "got_raw": "foo_text",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.625
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "foo_text",
+      "got": "foo_text",
+      "got_raw": "foo_text",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "\"tab: complete\")",
+      "got": "base_app._readline_bindings)",
+      "got_raw": "base_app._readline_bindings)",
+      "exact_match": false,
+      "code_bleu": 0.35930411196308426,
+      "edit_similarity": 0.18181818181818182
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "original_completer)",
+      "got": "original_completer)",
+      "got_raw": "original_completer)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "\"bind ^I rl_complete\")",
+      "got": "'libedit')",
+      "got_raw": "'libedit')",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "string",
+      "got": "string",
+      "got_raw": "string",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "TypeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "input_string",
+      "got": "input_string",
+      "got_raw": "input_string",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "expected_line",
+      "got": "expected_line",
+      "got_raw": "expected_line",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "orig_cmd2_style",
+      "got": "orig_cmd2_style",
+      "got_raw": "orig_cmd2_style",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "orig_rich_style",
+      "got": "orig_rich_style",
+      "got_raw": "orig_rich_style",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "expected_output",
+      "got": "expected_output",
+      "got_raw": "expected_output",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "str(excinfo.value)",
+      "got": "str(excinfo.value)",
+      "got_raw": "str(excinfo.value)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "orig_argparse_style",
+      "got": "orig_argparse_style",
+      "got_raw": "orig_argparse_style",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "theme[cmd2_style_key]",
+      "got": "Style(color=Color.CYAN)",
+      "got_raw": "Style(color=Color.CYAN)",
+      "exact_match": false,
+      "code_bleu": 0.13485111859503685,
+      "edit_similarity": 0.18181818181818182
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "theme[rich_style_key]",
+      "got": "theme[rich_style_key]",
+      "got_raw": "theme[rich_style_key]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "ru._from_ansi_wrapper.__func__",
+      "got": "Text.from_ansi",
+      "got_raw": "Text.from_ansi",
+      "exact_match": false,
+      "code_bleu": 0.2710806010829534,
+      "edit_similarity": 0.45454545454545453
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "out",
+      "got": "out",
+      "got_raw": "out",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "err",
+      "got": "err",
+      "got_raw": "err",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "out2",
+      "got": "out2",
+      "got_raw": "out2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "False",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "err[0]",
+      "got": "err",
+      "got_raw": "err",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "err[1]",
+      "got": "err",
+      "got_raw": "err",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "'help'",
+      "got": "'help'",
+      "got_raw": "'help'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "out[0]",
+      "got": "out",
+      "got_raw": "out",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "s2",
+      "got": "s2",
+      "got_raw": "s2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "\"Red\"",
+      "got": "\"Red\"",
+      "got_raw": "\"Red\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "\"Bold\"",
+      "got": "\"Bold\"",
+      "got_raw": "\"Bold\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "my_str",
+      "got": "my_str",
+      "got_raw": "my_str",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "\"Orange\"",
+      "got": "\"Orange\"",
+      "got_raw": "\"Orange\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "your_str",
+      "got": "your_str",
+      "got_raw": "your_str",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "stripped",
+      "got": "stripped",
+      "got_raw": "stripped",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "micro_cf",
+      "got": "micro_cf",
+      "got_raw": "micro_cf",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "cursor_up",
+      "got": "cursor_up",
+      "got_raw": "cursor_up",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "cursor_pos",
+      "got": "cursor_pos",
+      "got_raw": "cursor_pos",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "hide_cursor",
+      "got": "hide_cursor",
+      "got_raw": "hide_cursor",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "show_cursor",
+      "got": "show_cursor",
+      "got_raw": "show_cursor",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "HELLO_WORLD",
+      "got": "HELLO_WORLD",
+      "got_raw": "HELLO_WORLD",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "text[:width]",
+      "got": "text",
+      "got_raw": "text",
+      "exact_match": false,
+      "code_bleu": 0.11892071150027211,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "\"Custom RGB\"",
+      "got": "\"Custom RGB\"",
+      "got_raw": "\"Custom RGB\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "su.norm_fold(s2)",
+      "got": "s2",
+      "got_raw": "s2",
+      "exact_match": false,
+      "code_bleu": 0.09910059291689341,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "character * width",
+      "got": "'-' * width",
+      "got_raw": "'-' * width",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "\"Underline Green\"",
+      "got": "\"Underline Green\"",
+      "got_raw": "\"Underline Green\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "'\"' + my_str + '\"'",
+      "got": "'\"Hello World\"'",
+      "got_raw": "'\"Hello World\"'",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "17",
+      "got": "17",
+      "got_raw": "17",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "''",
+      "got": "'lower'",
+      "got_raw": "'lower'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "{}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "b''",
+      "got": "b''",
+      "got_raw": "b''",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "str",
+      "got": "str",
+      "got_raw": "str",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "'x'",
+      "got": "\"x\"",
+      "got_raw": "\"x\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "int",
+      "got": "int",
+      "got_raw": "int",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-cmd2/cmd2",
+      "expected": "ChildClass",
+      "got": "ChildClass",
+      "got_raw": "ChildClass",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "t[0]",
+      "got": "t.year",
+      "got_raw": "t.year",
+      "exact_match": false,
+      "code_bleu": 0.3388507513536918,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "t[1]",
+      "got": "t[1]",
+      "got_raw": "t[1]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "t[2]",
+      "got": "t[2]",
+      "got_raw": "t[2]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "t[3]",
+      "got": "t[3]",
+      "got_raw": "t[3]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "t[4]",
+      "got": "t[4]",
+      "got_raw": "t[4]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "d.day",
+      "got": "d.day",
+      "got_raw": "d.day",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "d.year",
+      "got": "d.year",
+      "got_raw": "d.year",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "d.hour",
+      "got": "d.hour",
+      "got_raw": "d.hour",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "p",
+      "got": "p",
+      "got_raw": "p",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "d1",
+      "got": "d1",
+      "got_raw": "d1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "d1",
+      "got": "d1",
+      "got_raw": "d1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "d2",
+      "got": "d2",
+      "got_raw": "d2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "d3",
+      "got": "d3",
+      "got_raw": "d3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "dt",
+      "got": "dt",
+      "got_raw": "dt",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "d1",
+      "got": "d2",
+      "got_raw": "d2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "d2",
+      "got": "d2",
+      "got_raw": "d2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "d3",
+      "got": "d3",
+      "got_raw": "d3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "tz",
+      "got": "tz",
+      "got_raw": "tz",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "t2",
+      "got": "t2",
+      "got_raw": "t2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "t3",
+      "got": "t3",
+      "got_raw": "t3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "dt2",
+      "got": "dt2",
+      "got_raw": "dt2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "it2",
+      "got": "it2",
+      "got_raw": "it2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "pi)",
+      "got": "pi)",
+      "got_raw": "pi)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "end",
+      "got": "end",
+      "got_raw": "end",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "5",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "7",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "7",
+      "got": "7",
+      "got_raw": "7",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "7",
+      "got": "7",
+      "got_raw": "7",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "9",
+      "got": "9",
+      "got_raw": "9",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "6",
+      "got": "6",
+      "got_raw": "6",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "2",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "1",
+      "got": "1970",
+      "got_raw": "1970",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "6",
+      "got": "6",
+      "got_raw": "6",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "3",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "7",
+      "got": "7",
+      "got_raw": "7",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "7",
+      "got": "7",
+      "got_raw": "7",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "9",
+      "got": "9",
+      "got_raw": "9",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "6",
+      "got": "6",
+      "got_raw": "6",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "7",
+      "got": "7",
+      "got_raw": "7",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "8",
+      "got": "8",
+      "got_raw": "8",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "9",
+      "got": "9",
+      "got_raw": "9",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "0",
+      "got": "9",
+      "got_raw": "9",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "2",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "1",
+      "got": "11",
+      "got_raw": "11",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "8",
+      "got": "8",
+      "got_raw": "8",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "1",
+      "got": "-1000001",
+      "got_raw": "-1000001",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "7",
+      "got": "6",
+      "got_raw": "6",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "4",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "5",
+      "got": "6",
+      "got_raw": "6",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "6",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "0",
+      "got": "12",
+      "got_raw": "12",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "4",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "6",
+      "got": "6",
+      "got_raw": "6",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "0",
+      "got": "11",
+      "got_raw": "11",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "3",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "1",
+      "got": "-2",
+      "got_raw": "-2",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "2",
+      "got": "-2",
+      "got_raw": "-2",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "1",
+      "got": "23",
+      "got_raw": "23",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "6",
+      "got": "6",
+      "got_raw": "6",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "0)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "7)",
+      "got": "7)",
+      "got_raw": "7)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "12",
+      "got": "12",
+      "got_raw": "12",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"1\"",
+      "got": "\"Q1\"",
+      "got_raw": "\"Q1\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"3\"",
+      "got": "\"3\"",
+      "got_raw": "\"3\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"7\"",
+      "got": "\"7\"",
+      "got_raw": "\"7\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"0\"",
+      "got": "\"28\"",
+      "got_raw": "\"28\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"6\"",
+      "got": "\"28\"",
+      "got_raw": "\"28\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"2\"",
+      "got": "\"1\"",
+      "got_raw": "\"1\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"J\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"3\"",
+      "got": "\"1\"",
+      "got_raw": "\"1\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"8\"",
+      "got": "\"3\"",
+      "got_raw": "\"3\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"3\"",
+      "got": "\"1\"",
+      "got_raw": "\"1\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"8\"",
+      "got": "\"3\"",
+      "got_raw": "\"3\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"fr\"",
+      "got": "\"fr\"",
+      "got_raw": "\"fr\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"25\"",
+      "got": "\"25\"",
+      "got_raw": "\"25\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"25\"",
+      "got": "\"25\"",
+      "got_raw": "\"25\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"po\"",
+      "got": "\"pon\"",
+      "got_raw": "\"pon\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"3.\"",
+      "got": "\"1\"",
+      "got_raw": "\"1\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"8.\"",
+      "got": "\"3.\"",
+      "got_raw": "\"3.\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"AM\"",
+      "got": "\"PM\"",
+      "got_raw": "\"PM\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"AM\"",
+      "got": "\"PM\"",
+      "got_raw": "\"PM\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"1\u00b0\"",
+      "got": "\"1\u00b0\"",
+      "got_raw": "\"1\u00b0\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"\u6570\u79d2\"",
+      "got": "\"\u6570\u79d2",
+      "got_raw": "\"\u6570\u79d2 \u5f8c\"",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"1\ucd08\"",
+      "got": "\"1\ucd08",
+      "got_raw": "\"1\ucd08 \uc804\"",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"2\ucd08\"",
+      "got": "\"1\ucd08\"",
+      "got_raw": "\"1\ucd08\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"3.\"",
+      "got": "\"1.\"",
+      "got_raw": "\"1.\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"8.\"",
+      "got": "\"3.\"",
+      "got_raw": "\"3.\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"zo\"",
+      "got": "\"zo\"",
+      "got_raw": "\"zo\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"3.\"",
+      "got": "\"1\"",
+      "got_raw": "\"1\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"8.\"",
+      "got": "\"3.\"",
+      "got_raw": "\"3.\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"AM\"",
+      "got": "\"p.m.\"",
+      "got_raw": "\"p.m.\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"29\"",
+      "got": "\"29\"",
+      "got_raw": "\"29\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"po\"",
+      "got": "\"pon\"",
+      "got_raw": "\"pon\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"AM\"",
+      "got": "\"AM\"",
+      "got_raw": "\"AM\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"29\"",
+      "got": "\"29\"",
+      "got_raw": "\"29\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"14\"",
+      "got": "\"14\"",
+      "got_raw": "\"14\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"15\"",
+      "got": "\"14:15\"",
+      "got_raw": "\"14:15\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7272727272727273
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"UTC\"",
+      "got": "\"UTC\"",
+      "got_raw": "\"UTC\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"UTC\"",
+      "got": "\"UTC\"",
+      "got_raw": "\"UTC\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"UTC\"",
+      "got": "\"UTC\"",
+      "got_raw": "\"UTC\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"UTC\"",
+      "got": "\"UTC\"",
+      "got_raw": "\"UTC\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"UTC\"",
+      "got": "\"UTC\"",
+      "got_raw": "\"UTC\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"srp\"",
+      "got": "\"srp\"",
+      "got_raw": "\"srp\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"29.\"",
+      "got": "\"29.\"",
+      "got_raw": "\"29.\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"28e\"",
+      "got": "\"28e\"",
+      "got_raw": "\"28e\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"dom\"",
+      "got": "\"dom\"",
+      "got_raw": "\"dom\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"ago\"",
+      "got": "\"ago\"",
+      "got_raw": "\"ago\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"28\u00b0\"",
+      "got": "\"28g\"",
+      "got_raw": "\"28g\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"28.\"",
+      "got": "\"28.\"",
+      "got_raw": "\"28.\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"28e\"",
+      "got": "\"28\"",
+      "got_raw": "\"28\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"29.\"",
+      "got": "\"29.\"",
+      "got_raw": "\"29.\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"sie\"",
+      "got": "\"sept.\"",
+      "got_raw": "\"sept.\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"1975\"",
+      "got": "\"1975\"",
+      "got_raw": "\"1975\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"1975\"",
+      "got": "\"1975\"",
+      "got_raw": "\"1975\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"dim.\"",
+      "got": "\"dim\"",
+      "got_raw": "\"dim\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"ao\u00fbt\"",
+      "got": "\"ao\u00fbt\"",
+      "got_raw": "\"ao\u00fbt\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"1 \u5206\u524d\"",
+      "got": "\"1 \u5206\u524d\"",
+      "got_raw": "\"1 \u5206\u524d\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"2 \u5206\u524d\"",
+      "got": "\"2 \u5206\u524d\"",
+      "got_raw": "\"2 \u5206\u524d\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"1 \u65e5\u524d\"",
+      "got": "\"1 \u65e5\u524d\"",
+      "got_raw": "\"1 \u65e5\u524d\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"1\ucd08 \uc804\"",
+      "got": "\"1\ucd08 \uc804\"",
+      "got_raw": "\"1\ucd08 \uc804\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"2\ucd08 \uc804\"",
+      "got": "\"2\ucd08 \uc804\"",
+      "got_raw": "\"2\ucd08 \uc804\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"1\ubd84 \uc804\"",
+      "got": "\"1\ubd84 \uc804\"",
+      "got_raw": "\"1\ubd84 \uc804\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"s\u00f8n.\"",
+      "got": "\"sun\"",
+      "got_raw": "\"sun\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7272727272727273
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"aug.\"",
+      "got": "\"aug.\"",
+      "got_raw": "\"aug.\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"aug.\"",
+      "got": "\"aug\"",
+      "got_raw": "\"aug\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"a.m.\"",
+      "got": "\"pm\"",
+      "got_raw": "\"pm\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"m\u00e5n.\"",
+      "got": "\"m\u00e5n\"",
+      "got_raw": "\"m\u00e5n\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"aug.\"",
+      "got": "\"aug.\"",
+      "got_raw": "\"aug.\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"test\"",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"+02:00\"",
+      "got": "\"Europe/Paris\"",
+      "got_raw": "\"Europe/Paris\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.18181818181818182
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"+06:00\"",
+      "got": "\"UTC\"",
+      "got_raw": "\"UTC\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"+00:00\"",
+      "got": "\"UTC\"",
+      "got_raw": "\"UTC\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"1 week\"",
+      "got": "\"1 week\"",
+      "got_raw": "\"1 week\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"-1 day\"",
+      "got": "\"1 day\"",
+      "got_raw": "\"1 day\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9333333333333333
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"1 week\"",
+      "got": "\"1 week\"",
+      "got_raw": "\"1 week\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "\"-1 day\"",
+      "got": "\"1 day ago\"",
+      "got_raw": "\"1 day ago\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7368421052631579
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "hash(p)",
+      "got": "hash(d)",
+      "got_raw": "hash(d)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "p.dst()",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "hash(d1)",
+      "got": "hash(d1)",
+      "got_raw": "hash(d1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "hash(d3)",
+      "got": "hash(d3)",
+      "got_raw": "hash(d3)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "dt.dst()",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "hash(dt)",
+      "got": "hash(dt)",
+      "got_raw": "hash(dt)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "d.ctime()",
+      "got": "d.ctime()",
+      "got_raw": "d.ctime()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "dt.date()",
+      "got": "dt.date()",
+      "got_raw": "dt.date()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "dt.time()",
+      "got": "time(12",
+      "got_raw": "time(12, 34, 56)",
+      "exact_match": false,
+      "code_bleu": 0.3567621345008163,
+      "edit_similarity": 0.625
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "hash(dt2)",
+      "got": "hash(dt2)",
+      "got_raw": "hash(dt2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "hash(dt3)",
+      "got": "hash(dt3)",
+      "got_raw": "hash(dt3)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "dt.date()",
+      "got": "dt.date()",
+      "got_raw": "dt.date()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "python-pendulum/pendulum",
+      "expected": "hash(dt1)",
+      "got": "hash(dt1)",
+      "got_raw": "hash(dt1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "str(ex.value)",
+      "got": "str(ex.value)",
+      "got_raw": "str(ex.value)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "luma.core.error.DeviceDisplayModeError)",
+      "got": "luma.core.error.DeviceDisplayModeError)",
+      "got_raw": "luma.core.error.DeviceDisplayModeError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "7",
+      "got": "7",
+      "got_raw": "7",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "1",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "start_frame() + [0xE0, 0, 0, 0] * 7 + end_frame(7))",
+      "got": "[0] * 7)",
+      "got_raw": "[0] * 7)",
+      "exact_match": false,
+      "code_bleu": 0.20203050891044216,
+      "edit_similarity": 0.2711864406779661
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "start_frame() + [0xE6, 0, 0, 0xFF] * 6 + end_frame(6))",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "start_frame() + [0xE4, 0x33, 0x22, 0x11] * 4 + end_frame(4))",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "8",
+      "got": "8",
+      "got_raw": "8",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "32",
+      "got": "16",
+      "got_raw": "16",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "22)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[12, 0] * 5)",
+      "got": "[0x00, 0x00, 0x00, 0x00, 0x00])",
+      "got_raw": "[0x00, 0x00, 0x00, 0x00, 0x00])",
+      "exact_match": false,
+      "code_bleu": 0.13065113298388567,
+      "edit_similarity": 0.32558139534883723
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[12, 1] * 3)",
+      "got": "[0x00] * 16 * 3)",
+      "got_raw": "[0x00] * 16 * 3)",
+      "exact_match": false,
+      "code_bleu": 0.3549481056010052,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[10, 6] * 6)",
+      "got": "[0x6B])",
+      "got_raw": "[0x6B])",
+      "exact_match": false,
+      "code_bleu": 0.21364350319811704,
+      "edit_similarity": 0.5263157894736842
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "AssertionError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5384615384615384
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[ call([11, 7]), call([9, 0]), call([15, 0]), call([10, 7]), call([1, 0]), call([2, 0]), call([3, 0]), call([4, 0]), call([5, 0]), call([6, 0]), call([7, 0]), call([8, 0]), call([12, 1]) ])",
+      "got": "[call(0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00)])",
+      "got_raw": "[call(0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00)])",
+      "exact_match": false,
+      "code_bleu": 0.036324051370285236,
+      "edit_similarity": 0.2671232876712329
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[ call([1, 0x81, 1, 0xFF]), call([2, 0x81, 2, 0x81]), call([3, 0x81, 3, 0x81]), call([4, 0x81, 4, 0x81]), call([5, 0x81, 5, 0x81]), call([6, 0x81, 6, 0x81]), call([7, 0x81, 7, 0x81]), call([8, 0xFF, 8, 0x81]) ])",
+      "got": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x",
+      "got_raw": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x",
+      "exact_match": false,
+      "code_bleu": 0.021000437459062837,
+      "edit_similarity": 0.005194805194805195
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[ call([1, 0x09, 1, 0x0F]), call([2, 0x09, 2, 0x09]), call([3, 0x09, 3, 0x09]), call([4, 0x09, 4, 0x09]), call([5, 0x09, 5, 0x09]), call([6, 0x09, 6, 0x09]), call([7, 0x09, 7, 0x09]), call([8, 0x0F, 8, 0x09]) ])",
+      "got": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x",
+      "got_raw": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x",
+      "exact_match": false,
+      "code_bleu": 0.021000437459062837,
+      "edit_similarity": 0.005194805194805195
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[ call([1, 0x00, 1, 0x00]), call([2, 0x00, 2, 0x00]), call([3, 0x00, 3, 0x00]), call([4, 0x00, 4, 0x00]), call([5, 0xFF, 5, 0xFF]), call([6, 0x80, 6, 0x01]), call([7, 0x80, 7, 0x01]), call([8, 0xFF, 8, 0xFF]) ])",
+      "got": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x",
+      "got_raw": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x",
+      "exact_match": false,
+      "code_bleu": 0.021000437459062837,
+      "edit_similarity": 0.005194805194805195
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[ call([1, 0xFF, 1, 0xFF]), call([2, 0x01, 2, 0x80]), call([3, 0x01, 3, 0x80]), call([4, 0xFF, 4, 0xFF]), call([5, 0x00, 5, 0x00]), call([6, 0x00, 6, 0x00]), call([7, 0x00, 7, 0x00]), call([8, 0x00, 8, 0x00]) ])",
+      "got": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x",
+      "got_raw": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x",
+      "exact_match": false,
+      "code_bleu": 0.021000437459062837,
+      "edit_similarity": 0.005194805194805195
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[ call([1, 0xF0, 1, 0x90]), call([2, 0x90, 2, 0x90]), call([3, 0x90, 3, 0x90]), call([4, 0x90, 4, 0x90]), call([5, 0x90, 5, 0x90]), call([6, 0x90, 6, 0x90]), call([7, 0x90, 7, 0x90]), call([8, 0x90, 8, 0xF0]) ])",
+      "got": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x",
+      "got_raw": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x",
+      "exact_match": false,
+      "code_bleu": 0.021000437459062837,
+      "edit_similarity": 0.005194805194805195
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[ call([1, 15, 1, 9, 1, 0, 1, 0]), call([2, 9, 2, 9, 2, 0, 2, 0]), call([3, 9, 3, 9, 3, 0, 3, 0]), call([4, 9, 4, 9, 4, 0, 4, 0]), call([5, 9, 5, 9, 5, 0, 5, 0]), call([6, 9, 6, 9, 6, 0, 6, 0]), call([7, 9, 7, 9, 7, 0, 7, 0]), call([8, 9, 8, 15, 8, 0, 8, 0]) ])",
+      "got": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x",
+      "got_raw": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x",
+      "exact_match": false,
+      "code_bleu": 0.013094390415650945,
+      "edit_similarity": 0.004597701149425287
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[ call([11, 7, 11, 7]), call([9, 0, 9, 0]), call([15, 0, 15, 0]), call([10, 7, 10, 7]), call([1, 0, 1, 0]), call([2, 0, 2, 0]), call([3, 0, 3, 0]), call([4, 0, 4, 0]), call([5, 0, 5, 0]), call([6, 0, 6, 0]), call([7, 0, 7, 0]), call([8, 0, 8, 0]), call([12, 1, 12, 1]) ])",
+      "got": "[call(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,",
+      "got_raw": "[call(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,",
+      "exact_match": false,
+      "code_bleu": 0.020702192795404424,
+      "edit_similarity": 0.004975124378109453
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[ call([1, 0x80, 1, 0xFF, 1, 0x01, 1, 0xFF]), call([2, 0x80, 2, 0x80, 2, 0x01, 2, 0x01]), call([3, 0x80, 3, 0x80, 3, 0x01, 3, 0x01]), call([4, 0x80, 4, 0x80, 4, 0x01, 4, 0x01]), call([5, 0x80, 5, 0x80, 5, 0x01, 5, 0x01]), call([6, 0x80, 6, 0x80, 6, 0x01, 6, 0x01]), call([7, 0x80, 7, 0x80, 7, 0x01, 7, 0x01]), call([8, 0xFF, 8, 0x80, 8, 0xFF, 8, 0x01]) ])",
+      "got": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x",
+      "got_raw": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x",
+      "exact_match": false,
+      "code_bleu": 0.013094390415650945,
+      "edit_similarity": 0.003780718336483932
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "str(ex.value)",
+      "got": "str(ex.value)",
+      "got_raw": "str(ex.value)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "OverflowError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.56
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "neoseg.device.image)",
+      "got": "neoseg.image)",
+      "got_raw": "neoseg.image)",
+      "exact_match": false,
+      "code_bleu": 0.5065571237677283,
+      "edit_similarity": 0.7878787878787878
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "luma.core.error.DeviceDisplayModeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.13432124147794275,
+      "edit_similarity": 0.36
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[0x7f, 0x7f]",
+      "got": "[\"B\", \"B\"]",
+      "got_raw": "[\"B\", \"B\"]",
+      "exact_match": false,
+      "code_bleu": 0.32466791547509893,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[0x7f, 0x08, 0x7f]",
+      "got": "['B', 'B', '_']",
+      "got_raw": "['B', 'B', '_']",
+      "exact_match": false,
+      "code_bleu": 0.23356898886410002,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[0x80, 0x67, 0x7e, 0x47]",
+      "got": "[\"P\", \"D\", \"O\", \"F\"]",
+      "got_raw": "[\"P\", \"D\", \"O\", \"F\"]",
+      "exact_match": false,
+      "code_bleu": 0.18575057999133596,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[0x76, 0x1d | 0x80, 0x80, 0x80]",
+      "got": "[\"No...\"]",
+      "got_raw": "[\"No...\"]",
+      "exact_match": false,
+      "code_bleu": 0.13636363636363633,
+      "edit_similarity": 0.1
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[0x79, 0x80, 0x30, 0x33, 0x30, 0x5b, 0x7b]",
+      "got": "[\"3\", \"1\", \"4\", \"1\", \"5\", \"9\"]",
+      "got_raw": "[\"3\", \"1\", \"4\", \"1\", \"5\", \"9\"]",
+      "exact_match": false,
+      "code_bleu": 0.11707475569704556,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[0x79 | 0x80, 0x30, 0x33, 0x30, 0x5b, 0x7b]",
+      "got": "[\"3\", \"1\", \"4\", \"1\", \"5\", \"9\"]",
+      "got_raw": "[\"3\", \"1\", \"4\", \"1\", \"5\", \"9\"]",
+      "exact_match": false,
+      "code_bleu": 0.11707475569704556,
+      "edit_similarity": 0.3561643835616438
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[0x6d, 0x7b | 0x80, 0x30, 0x6d, 0x63, 0x4e]",
+      "got": "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0",
+      "got_raw": "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0",
+      "exact_match": false,
+      "code_bleu": 0.01902591407689958,
+      "edit_similarity": 0.17543859649122806
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[0x30, 0x6d, 0x70, 0x80, 0x7e, 0x80, 0x7e, 0x80, 0x30]",
+      "got": "[1, 2, 7, 0, 0, 1]",
+      "got_raw": "[1, 2, 7, 0, 0, 1]",
+      "exact_match": false,
+      "code_bleu": 0.09242743870819387,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[0x00, 0x00, 0x5b, 0x6d, 0x5b, 0x7b, 0x6d, 0x7e | 0x80]",
+      "got": "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]",
+      "got_raw": "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]",
+      "exact_match": false,
+      "code_bleu": 0.08687475782716618,
+      "edit_similarity": 0.49411764705882355
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[0x30, 0x6d, 0x70 | 0x80, 0x7e | 0x80, 0x7e | 0x80, 0x30]",
+      "got": "[1, 2, 7, 0, 0, 1]",
+      "got_raw": "[1, 2, 7, 0, 0, 1]",
+      "exact_match": false,
+      "code_bleu": 0.09242743870819387,
+      "edit_similarity": 0.21333333333333335
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[0x37, 0x6f, 0x06, 0x06, 0x1d, 0x00, 0x14, 0x1d, 0x05, 0x06, 0x3d]",
+      "got": "['H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd']",
+      "got_raw": "['H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd']",
+      "exact_match": false,
+      "code_bleu": 0.08449686085328642,
+      "edit_similarity": 0.38016528925619836
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "16",
+      "got": "16",
+      "got_raw": "16",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[0x72] + [0] * 256 * 3)",
+      "got": "0)",
+      "got_raw": "0)",
+      "exact_match": false,
+      "code_bleu": 0.09910059291689341,
+      "edit_similarity": 0.16
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[0x72] + [0xFF] * 256 * 3)",
+      "got": "[0x00",
+      "got_raw": "[0x00, 0x00, 0x00, 0x00])",
+      "exact_match": false,
+      "code_bleu": 0.08954749431862849,
+      "edit_similarity": 0.25806451612903225
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[0x72] + [0x6B] * 256 * 3)",
+      "got": "[0x6B])",
+      "got_raw": "[0x6B])",
+      "exact_match": false,
+      "code_bleu": 0.2357022603955158,
+      "edit_similarity": 0.42424242424242425
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[0x72] + get_json_data('demo_unicornhathd'))",
+      "got": "b\"000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+      "got_raw": "b\"000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+      "exact_match": false,
+      "code_bleu": 0.12663928094193208,
+      "edit_similarity": 0.011627906976744186
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[0x72] + get_json_data('demo_unicornhathd_alphablend'))",
+      "got": "b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00",
+      "got_raw": "b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00",
+      "exact_match": false,
+      "code_bleu": 0.011696906573546547,
+      "edit_similarity": 0.017777777777777778
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "7",
+      "got": "7",
+      "got_raw": "7",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "32",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "0x6B)",
+      "got": "0x6B)",
+      "got_raw": "0x6B)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "expected)",
+      "got": "expected)",
+      "got_raw": "expected)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "str(ex.value)",
+      "got": "str(ex.value)",
+      "got_raw": "str(ex.value)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "RuntimeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[ call(chan, i, 0x000000) for i in range(7)])",
+      "got": "[call(leds, 0, 0)])",
+      "got_raw": "[call(leds, 0, 0)])",
+      "exact_match": false,
+      "code_bleu": 0.19952600730504763,
+      "edit_similarity": 0.40625
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[ call(chan, i, 0x000000) for i in range(32)])",
+      "got": "[call(leds, 0, 0, 0, 0)])",
+      "got_raw": "[call(leds, 0, 0, 0, 0)])",
+      "exact_match": false,
+      "code_bleu": 0.19648596703776508,
+      "edit_similarity": 0.4225352112676056
+    },
+    {
+      "repo": "rm-hull/luma.led_matrix",
+      "expected": "[ call(chan, 0, 0xFF0000), call(chan, 1, 0xFF0000), call(chan, 2, 0xFF0000), call(chan, 3, 0xFF0000), call(chan, 4, 0xFF0000), call(chan, 5, 0x000000), call(chan, 6, 0x000000), call(chan, 7, 0xFF0000), call(chan, 8, 0xFF0000), call(chan, 9, 0x000000), call(chan, 10, 0x000000), call(chan, 11, 0xFF0000), call(chan, 12, 0xFF0000), call(chan, 13, 0xFF0000), call(chan, 14, 0xFF0000), call(chan, 15, 0xFF0000), ])",
+      "got": "[call(leds, 0, 0, 0, 0, 0, 0, 0, 0)])",
+      "got_raw": "[call(leds, 0, 0, 0, 0, 0, 0, 0, 0)])",
+      "exact_match": false,
+      "code_bleu": 0.028682259444772533,
+      "edit_similarity": 0.013422818791946308
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "np.float32",
+      "got": "np.float32",
+      "got_raw": "np.float32",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "llvm_pred)",
+      "got": "llvm_pred)",
+      "got_raw": "llvm_pred)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "llvm.predict(data, n_jobs=2))",
+      "got": "lgb.predict(data))",
+      "got_raw": "lgb.predict(data))",
+      "exact_match": false,
+      "code_bleu": 0.37838408204632035,
+      "edit_similarity": 0.6808510638297872
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "[results[res_idx]]",
+      "got": "res_idx",
+      "got_raw": "res_idx",
+      "exact_match": false,
+      "code_bleu": 0.09910059291689341,
+      "edit_similarity": 0.56
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "list(df2[\"C1\"].cat.codes)",
+      "got": "list(df2[\"C1\"].cat.codes)",
+      "got_raw": "list(df2[\"C1\"].cat.codes)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "list(df2[\"C2\"].cat.codes)",
+      "got": "list(df2[\"C2\"].cat.codes)",
+      "got_raw": "list(df2[\"C2\"].cat.codes)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "lgbm_model.predict(data))",
+      "got": "lightgbm_model.predict(data))",
+      "got_raw": "lightgbm_model.predict(data))",
+      "exact_match": false,
+      "code_bleu": 0.8408964152537145,
+      "edit_similarity": 0.9259259259259259
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "lgbm_model.predict([input])",
+      "got": "lgbm_model.predict(input)",
+      "got_raw": "lgbm_model.predict(input)",
+      "exact_match": false,
+      "code_bleu": 0.5015552287323165,
+      "edit_similarity": 0.9615384615384616
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "Path(model_path).read_text()",
+      "got": "model_path",
+      "got_raw": "model_path",
+      "exact_match": false,
+      "code_bleu": 0.07432544468767006,
+      "edit_similarity": 0.5263157894736842
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "lgbm_model.predict(tests_data))",
+      "got": "label)",
+      "got_raw": "label)",
+      "exact_match": false,
+      "code_bleu": 0.153509990260506,
+      "edit_similarity": 0.2702702702702703
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "stdout",
+      "got": "stdout",
+      "got_raw": "stdout",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "lgbm_model.predict(data, n_jobs=2))",
+      "got": "iniest_single_tree_123132_XXX(data)",
+      "got_raw": "iniest_single_tree_123132_XXX(data)",
+      "exact_match": false,
+      "code_bleu": 0.18477554478349628,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "lgbm_model.predict(NYC_data[:1000], n_jobs=2))",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "1",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "5",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "[]",
+      "got": "[[\"a\"",
+      "got_raw": "[[\"a\", \"b\", \"c\"], [\"b\", \"c\", \"d\"], [\"w\", \"x\", \"y\", \"z\"]]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "orig)",
+      "got": "orig)",
+      "got_raw": "orig)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "orig.dtype",
+      "got": "orig.dtype",
+      "got_raw": "orig.dtype",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "lgbm_model.predict(df))",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "lgbm_model.predict(sliced))",
+      "got": "data)",
+      "got_raw": "data)",
+      "exact_match": false,
+      "code_bleu": 0.153509990260506,
+      "edit_similarity": 0.1875
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "[ [\"a\", \"b\", \"c\"], [\"b\", \"c\", \"d\"], [\"w\", \"x\", \"y\", \"z\"], ]",
+      "got": "[[\"a\", \"b\", \"c\"], [\"b\", \"c\", \"d\"], [\"w\", \"x\", \"y\", \"z\"]]",
+      "got_raw": "[[\"a\", \"b\", \"c\"], [\"b\", \"c\", \"d\"], [\"w\", \"x\", \"y\", \"z\"]]",
+      "exact_match": false,
+      "code_bleu": 0.9363055597212666,
+      "edit_similarity": 0.9739130434782609
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "res)",
+      "got": "tiniest_llvm.predict([3",
+      "got_raw": "tiniest_llvm.predict([3 * [0.0], 3 * [1.0], 3 * [-1.0]]))",
+      "exact_match": false,
+      "code_bleu": 0.18575057999133598,
+      "edit_similarity": 0.14814814814814814
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "lgbm.predict(arr)",
+      "got": "lgbm.predict(arr)",
+      "got_raw": "lgbm.predict(arr)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "err1.value.args[0]",
+      "got": "str(err1.value)",
+      "got_raw": "str(err1.value)",
+      "exact_match": false,
+      "code_bleu": 0.30835021267538976,
+      "edit_similarity": 0.6060606060606061
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "err2.value.args[0]",
+      "got": "err2.value.args[0]",
+      "got_raw": "err2.value.args[0]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "pure_cat_llvm.predict([3 * [0.0], 3 * [1.0], 3 * [-1.0]]))",
+      "got": "iniest_llvm.predict([3 * [0.0], 3 * [1.0], 3 * [-1.0]])",
+      "got_raw": "iniest_llvm.predict([3 * [0.0], 3 * [1.0], 3 * [-1.0]])",
+      "exact_match": false,
+      "code_bleu": 0.9222082305707098,
+      "edit_similarity": 0.9026548672566371
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "lgbm_model.predict(data))",
+      "got": "lgbm_model.predict(data))",
+      "got_raw": "lgbm_model.predict(data))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "llvm_model.predict(data))",
+      "got": "llvm_model.predict(data))",
+      "got_raw": "llvm_model.predict(data))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "llvm_model.predict(X))",
+      "got": "0.5)",
+      "got_raw": "0.5)",
+      "exact_match": false,
+      "code_bleu": 0.153509990260506,
+      "edit_similarity": 0.15384615384615385
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "llvm_model.predict(data))",
+      "got": "llvm_model.predict(data))",
+      "got_raw": "llvm_model.predict(data))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "NYC_lgbm.predict(data))",
+      "got": "data)",
+      "got_raw": "data)",
+      "exact_match": false,
+      "code_bleu": 0.20203050891044216,
+      "edit_similarity": 0.35714285714285715
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "4",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "12",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "18",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "95",
+      "got": "[1",
+      "got_raw": "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "np.float64",
+      "got": "lgbm_result.dtype",
+      "got_raw": "lgbm_result.dtype",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.14814814814814814
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "np.float32",
+      "got": "np.float32",
+      "got_raw": "np.float32",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "llvm_result)",
+      "got": "llvm_result)",
+      "got_raw": "llvm_result)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "lightgbm_model.num_trees()",
+      "got": "lightgbm_model.num_trees()",
+      "got_raw": "lightgbm_model.num_trees()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "llvm.predict(X, n_jobs=2))",
+      "got": "lgbm.predict(X))",
+      "got_raw": "lgbm.predict(X))",
+      "exact_match": false,
+      "code_bleu": 0.37838408204632035,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "lightgbm_model.num_feature()",
+      "got": "lightgbm_model.num_feature()",
+      "got_raw": "lightgbm_model.num_feature()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "llvm.num_model_per_iteration()",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "lgbm_model.predict([input_data]))",
+      "got": "input_data)",
+      "got_raw": "input_data)",
+      "exact_match": false,
+      "code_bleu": 0.1321341238891912,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "lightgbm_model.predict(input_data))",
+      "got": "(lightgbm_model.predict(input_data))",
+      "got_raw": "(lightgbm_model.predict(input_data))",
+      "exact_match": false,
+      "code_bleu": 0.8633400213704505,
+      "edit_similarity": 0.9859154929577465
+    },
+    {
+      "repo": "siboehm/lleaves",
+      "expected": "lightgbm_model.num_model_per_iteration()",
+      "got": "0.0",
+      "got_raw": "0.0",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.046511627906976744
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "test_session",
+      "got": "test_session",
+      "got_raw": "test_session",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "YTMusicUserError, match=\"Invalid auth\")",
+      "got": "YTMusicUserError)",
+      "got_raw": "YTMusicUserError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.6071428571428571
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "second_json",
+      "got": "second_json",
+      "got_raw": "second_json",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "second_token",
+      "got": "second_token",
+      "got_raw": "second_token",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "first_expire",
+      "got": "first_expire",
+      "got_raw": "first_expire",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "time.time() + 60",
+      "got": "first_expire",
+      "got_raw": "first_expire",
+      "exact_match": false,
+      "code_bleu": 0.07142857142857142,
+      "edit_similarity": 0.21428571428571427
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "YTMusicUserError)",
+      "got": "YTMusicUserError)",
+      "got_raw": "YTMusicUserError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "AuthType.OAUTH_CUSTOM_FULL",
+      "got": "AuthType.OAUTH_CUSTOM_FULL",
+      "got_raw": "AuthType.OAUTH_CUSTOM_FULL",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "AuthType.OAUTH_CUSTOM_CLIENT",
+      "got": "AuthType.BROWSER",
+      "got_raw": "AuthType.BROWSER",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.5454545454545454
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "3",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "9",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "7",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "5",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "3",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "4",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "5",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "25",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "50",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "40",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "5",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "21",
+      "got": "21",
+      "got_raw": "21",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "10",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "34",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "46",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "456",
+      "got": "500",
+      "got_raw": "500",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "100",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "255",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "False",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "playlist",
+      "got": "playlist",
+      "got_raw": "playlist",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "tracks_len",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "5",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "9",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "10",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "20",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "80",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "150",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "100",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "5",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "3",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "4",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "1",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "10",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "20",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "25",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "40",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "409",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "ResponseStatus.SUCCEEDED",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "config.getint(\"limits\", \"library_upload_albums\")",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.0625,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "config.getint(\"limits\", \"library_upload_artists\")",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.0625,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "Exception, match=\"The provided file does not exist\")",
+      "got": "YTMusicUserError)",
+      "got_raw": "YTMusicUserError)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.14492753623188406
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "YTMusicUserError, match=\"larger than the limit of 300MB\")",
+      "got": "YTMusicUserError)",
+      "got_raw": "YTMusicUserError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.4594594594594595
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "Exception, match=\"The provided file type is not supported\")",
+      "got": "YTMusicUserError)",
+      "got_raw": "YTMusicUserError)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.15789473684210525
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "90",
+      "got": "90",
+      "got_raw": "90",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "45",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "25",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "config.getint(\"playlists\", \"own_length\")",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.0625,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "config.getint(\"albums\", \"album_track_length\")",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.0625,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "sigma67/ytmusicapi",
+      "expected": "Exception, match=\"No content returned by the server\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.1875
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "5",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "0",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "res.data",
+      "got": "res",
+      "got_raw": "res",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.5454545454545454
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "res.data",
+      "got": "res",
+      "got_raw": "res",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.5454545454545454
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "user.email",
+      "got": "credentials.email",
+      "got_raw": "credentials.email",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.5185185185185185
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "user.email",
+      "got": "credentials.email",
+      "got_raw": "credentials.email",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.5185185185185185
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "err[\"code\"]",
+      "got": "400",
+      "got_raw": "400",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "dict[str, str]",
+      "got": "Dict[str, str]",
+      "got_raw": "Dict[str, str]",
+      "exact_match": false,
+      "code_bleu": 0.8091067115702212,
+      "edit_similarity": 0.9285714285714286
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "client.headers",
+      "got": "client.headers",
+      "got_raw": "client.headers",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "client.headers",
+      "got": "client.headers",
+      "got_raw": "client.headers",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "credentials.email",
+      "got": "credentials.email",
+      "got_raw": "credentials.email",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "credentials.email",
+      "got": "credentials.email",
+      "got_raw": "credentials.email",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "{\"test\": \"data\"}",
+      "got": "{\"test\": \"data\"}",
+      "got_raw": "{\"test\": \"data\"}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "{\"test\": \"data\"}",
+      "got": "{\"test\": \"data\"}",
+      "got_raw": "{\"test\": \"data\"}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "{\"name\": \"Michael\"}",
+      "got": "{\"name\": \"John\"}",
+      "got_raw": "{\"name\": \"John\"}",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.7428571428571429
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "{\"name\": \"Michael\"}",
+      "got": "{\"name\": \"John\"}",
+      "got_raw": "{\"name\": \"John\"}",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.7428571428571429
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "{\"message\": \"success\"}",
+      "got": "{\"message\": \"success\"}",
+      "got_raw": "{\"message\": \"success\"}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "{\"message\": \"success\"}",
+      "got": "{\"message\": \"success\"}",
+      "got_raw": "{\"message\": \"success\"}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "{\"id\": 2, \"name\": \"Jane\"}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.1321341238891912,
+      "edit_similarity": 0.14814814814814814
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "{\"id\": 2, \"name\": \"Jane\"}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.1321341238891912,
+      "edit_similarity": 0.14814814814814814
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "{\"id\": 1, \"name\": \"Michael\"}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.1321341238891912,
+      "edit_similarity": 0.13333333333333333
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "{\"id\": 1, \"name\": \"Michael\"}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.1321341238891912,
+      "edit_similarity": 0.13333333333333333
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "[{\"id\": 2, \"room_name\": \"Topaz\"}]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.1081097377275201,
+      "edit_similarity": 0.11428571428571428
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "[{\"id\": 2, \"room_name\": \"Topaz\"}]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.1081097377275201,
+      "edit_similarity": 0.11428571428571428
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "[{\"id\": 1, \"room_name\": \"Emerald\"}]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.1081097377275201,
+      "edit_similarity": 0.10810810810810811
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "str",
+      "got": "str",
+      "got_raw": "str",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "res",
+      "got": "\"EXPLAIN",
+      "got_raw": "\"EXPLAIN (text)\"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "res",
+      "got": "\"EXPLAIN",
+      "got_raw": "\"EXPLAIN (ANALYZE, VERBOSE, SETTINGS, BUFFERS, WALENABLED)\"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "key",
+      "got": "key",
+      "got_raw": "key",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "key",
+      "got": "key",
+      "got_raw": "key",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "bool",
+      "got": "bool",
+      "got_raw": "bool",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "code",
+      "got": "code",
+      "got_raw": "code",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "path",
+      "got": "path",
+      "got_raw": "path",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "path",
+      "got": "path",
+      "got_raw": "path",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "email",
+      "got": "email",
+      "got_raw": "email",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "email",
+      "got": "email",
+      "got_raw": "email",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "path)",
+      "got": "request.assert_called_once()",
+      "got_raw": "request.assert_called_once()",
+      "exact_match": false,
+      "code_bleu": 0.2730120862709067,
+      "edit_similarity": 0.12121212121212122
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "path)",
+      "got": "request.assert_called_once()",
+      "got_raw": "request.assert_called_once()",
+      "exact_match": false,
+      "code_bleu": 0.2730120862709067,
+      "edit_similarity": 0.12121212121212122
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "emails",
+      "got": "emails",
+      "got_raw": "emails",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "emails",
+      "got": "emails",
+      "got_raw": "emails",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "status",
+      "got": "status",
+      "got_raw": "status",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "copied",
+      "got": "copied",
+      "got_raw": "copied",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "params",
+      "got": "URL(expected_url).query",
+      "got_raw": "URL(expected_url).query",
+      "exact_match": false,
+      "code_bleu": 0.18575057999133598,
+      "edit_similarity": 0.13793103448275862
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "copied",
+      "got": "copied",
+      "got_raw": "copied",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "params",
+      "got": "URL(expected_url).query",
+      "got_raw": "URL(expected_url).query",
+      "exact_match": false,
+      "code_bleu": 0.18575057999133598,
+      "edit_similarity": 0.13793103448275862
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "message",
+      "got": "message",
+      "got_raw": "message",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "message",
+      "got": "message",
+      "got_raw": "message",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "callback",
+      "got": "callback",
+      "got_raw": "callback",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "client_id",
+      "got": "client_id",
+      "got_raw": "client_id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "client_id",
+      "got": "client_id",
+      "got_raw": "client_id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "valid_url",
+      "got": "valid_url",
+      "got_raw": "valid_url",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "valid_url",
+      "got": "valid_url",
+      "got_raw": "valid_url",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "valid_url",
+      "got": "valid_url",
+      "got_raw": "valid_url",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "APIError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "APIError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "bucket_id",
+      "got": "bucket_id",
+      "got_raw": "bucket_id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "bucket_id",
+      "got": "bucket_id",
+      "got_raw": "bucket_id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "actual_url",
+      "got": "str(actual_url)",
+      "got_raw": "str(actual_url)",
+      "exact_match": false,
+      "code_bleu": 0.35930411196308426,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "actual_url",
+      "got": "str(actual_url)",
+      "got_raw": "str(actual_url)",
+      "exact_match": false,
+      "code_bleu": 0.35930411196308426,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"*\"",
+      "got": "\"count\"",
+      "got_raw": "\"count\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"2\"",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"0\"",
+      "got": "\"0\"",
+      "got_raw": "\"0\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"1\"",
+      "got": "\"1\"",
+      "got_raw": "\"1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"*\"",
+      "got": "\"count\"",
+      "got_raw": "\"count\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"2\"",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"0\"",
+      "got": "\"0\"",
+      "got_raw": "\"0\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"1\"",
+      "got": "\"1\"",
+      "got_raw": "\"1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"any\"",
+      "got": "\"any\"",
+      "got_raw": "\"any\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"GET\"",
+      "got": "\"GET\"",
+      "got_raw": "\"GET\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"GET\"",
+      "got": "\"GET\"",
+      "got_raw": "\"GET\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"GET\"",
+      "got": "\"GET\"",
+      "got_raw": "\"GET\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"GET\"",
+      "got": "\"GET\"",
+      "got_raw": "\"GET\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"1.0\"",
+      "got": "\"1.0\"",
+      "got_raw": "\"1.0\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"1.0\"",
+      "got": "\"1.0\"",
+      "got_raw": "\"1.0\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"apple\"",
+      "got": "\"apple\"",
+      "got_raw": "\"apple\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"apple\"",
+      "got": "\"apple\"",
+      "got_raw": "\"apple\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"eq.val\"",
+      "got": "\"val\"",
+      "got_raw": "\"val\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7692307692307693
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"x=eq.a\"",
+      "got": "\"x=a\"",
+      "got_raw": "\"x=a\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7692307692307693
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"x=gt.a\"",
+      "got": "\"{'x':",
+      "got_raw": "\"{'x': 'a'}\"",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"x=cs.a\"",
+      "got": "\"{'x':",
+      "got_raw": "\"{'x': 'a'}\"",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"x=is.a\"",
+      "got": "\"{'x':",
+      "got_raw": "\"{'x': 'a'}\"",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"eq.val\"",
+      "got": "\"val\"",
+      "got_raw": "\"val\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7692307692307693
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"x=eq.a\"",
+      "got": "\"{'x':",
+      "got_raw": "\"{'x': 'a'}\"",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"x=gt.a\"",
+      "got": "\"{'x':",
+      "got_raw": "\"{'x': 'a'}\"",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"x=cs.a\"",
+      "got": "\"{'x':",
+      "got_raw": "\"{'x': 'a'}\"",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"x=is.a\"",
+      "got": "\"x=a\"",
+      "got_raw": "\"x=a\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7692307692307693
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"grapes\"",
+      "got": "\"grapes\"",
+      "got_raw": "\"grapes\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"grapes\"",
+      "got": "\"grapes\"",
+      "got_raw": "\"grapes\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"x=neq.a\"",
+      "got": "\"{'x':",
+      "got_raw": "\"{'x': 'a'}\"",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.26666666666666666
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"x=gte.a\"",
+      "got": "\"{'x':",
+      "got_raw": "\"{'x': 'a'}\"",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.26666666666666666
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"x=neq.a\"",
+      "got": "\"{'x':",
+      "got_raw": "\"{'x': 'a'}\"",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.26666666666666666
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"x=gte.a\"",
+      "got": "\"{'x':",
+      "got_raw": "\"{'x': 'a'}\"",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.26666666666666666
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"user123\"",
+      "got": "\"user123\"",
+      "got_raw": "\"user123\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"bucket1\"",
+      "got": "\"bucket1\"",
+      "got_raw": "\"bucket1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"bucket2\"",
+      "got": "\"bucket2\"",
+      "got_raw": "\"bucket2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"bucket\")",
+      "got": "\"/buckets\")",
+      "got_raw": "\"/buckets\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.9
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"bucket1\"",
+      "got": "\"bucket1\"",
+      "got_raw": "\"bucket1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"bucket2\"",
+      "got": "\"bucket2\"",
+      "got_raw": "\"bucket2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"bucket\")",
+      "got": "\"/buckets\")",
+      "got_raw": "\"/buckets\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.9
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"eu-west-1\"",
+      "got": "\"eu-west-1\"",
+      "got_raw": "\"eu-west-1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"eu-west-2\"",
+      "got": "\"eu-west-2\"",
+      "got_raw": "\"eu-west-2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"eu-west-3\"",
+      "got": "\"eu-west-3\"",
+      "got_raw": "\"eu-west-3\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"sa-east-1\"",
+      "got": "\"sa-east-1\"",
+      "got_raw": "\"sa-east-1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"us-east-1\"",
+      "got": "\"us-east-1\"",
+      "got_raw": "\"us-east-1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"us-east-1\"",
+      "got": "\"us-east-1\"",
+      "got_raw": "\"us-east-1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"Aggregate\"",
+      "got": "\"NestedLoop\"",
+      "got_raw": "\"NestedLoop\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.43478260869565216
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"Aggregate\"",
+      "got": "\"NestedLoop\"",
+      "got_raw": "\"NestedLoop\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.43478260869565216
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"new_value\"",
+      "got": "\"value\"",
+      "got_raw": "\"value\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7777777777777778
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"text/plain\"",
+      "got": "\"text/plain;",
+      "got_raw": "\"text/plain; charset=utf-8\"",
+      "exact_match": false,
+      "code_bleu": 0.2295748846661433,
+      "edit_similarity": 0.9166666666666666
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"text/plain\"",
+      "got": "\"text/plain;",
+      "got_raw": "\"text/plain; charset=utf-8\"",
+      "exact_match": false,
+      "code_bleu": 0.2295748846661433,
+      "edit_similarity": 0.9166666666666666
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"test-owner\"",
+      "got": "\"test-owner\"",
+      "got_raw": "\"test-owner\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"test-owner\"",
+      "got": "\"test-owner\"",
+      "got_raw": "\"test-owner\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"Test Bucket\"",
+      "got": "\"Test Bucket\"",
+      "got_raw": "\"Test Bucket\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"Test Bucket\"",
+      "got": "\"Test Bucket\"",
+      "got_raw": "\"Test Bucket\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"my-app/0.0.1\"",
+      "got": "\"my-app/0.0.1\"",
+      "got_raw": "\"my-app/0.0.1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"my-app/0.0.1\"",
+      "got": "\"my-app/0.0.1\"",
+      "got_raw": "\"my-app/0.0.1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"my-app/0.0.1\"",
+      "got": "\"my-app/0.0.1\"",
+      "got_raw": "\"my-app/0.0.1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"my-app/0.0.1\"",
+      "got": "\"my-app/0.0.1\"",
+      "got_raw": "\"my-app/0.0.1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"my-app/0.0.1\"",
+      "got": "\"my-app/0.0.1\"",
+      "got_raw": "\"my-app/0.0.1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"Bearer s3cr3t\"",
+      "got": "\"Bearer s3cr3t\"",
+      "got_raw": "\"Bearer s3cr3t\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"Bearer s3cr3t\"",
+      "got": "\"Bearer s3cr3t\"",
+      "got_raw": "\"Bearer s3cr3t\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"/example_table\"",
+      "got": "\"/example_table\"",
+      "got_raw": "\"/example_table\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"x=like.%25a%25\"",
+      "got": "\"x",
+      "got_raw": "\"x LIKE '%a%'\"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"/example_table\"",
+      "got": "\"/example_table\"",
+      "got_raw": "\"/example_table\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"/example_table\"",
+      "got": "\"/example_table\"",
+      "got_raw": "\"/example_table\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"x=like.%25a%25\"",
+      "got": "\"{'x':",
+      "got_raw": "\"{'x': 'a'}\"",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.18181818181818182
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"/example_table\"",
+      "got": "\"/example_table\"",
+      "got_raw": "\"/example_table\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "b\"binary content\"",
+      "got": "b\"binary content\"",
+      "got_raw": "b\"binary content\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "b\"binary content\"",
+      "got": "b\"binary content\"",
+      "got_raw": "b\"binary content\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"x=lte.a&x=gte.b\"",
+      "got": "\"{'x':",
+      "got_raw": "\"{'x': ['a', 'b']}\"",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.17391304347826086
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "\"x=ilike.%25a%25\"",
+      "got": "\"{'x':",
+      "got_raw": "\"{'x': 'a%'}\"",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.17391304347826086
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "Timeout(10)",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "Timeout(30)",
+      "got": "Timeout(30)",
+      "got_raw": "Timeout(30)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "Timeout(10)",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "Timeout(30)",
+      "got": "Timeout(30)",
+      "got_raw": "Timeout(30)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "info.keys()",
+      "got": "info",
+      "got_raw": "info",
+      "exact_match": false,
+      "code_bleu": 0.11892071150027211,
+      "edit_similarity": 0.5333333333333333
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "info.keys()",
+      "got": "info",
+      "got_raw": "info",
+      "exact_match": false,
+      "code_bleu": 0.11892071150027211,
+      "edit_similarity": 0.5333333333333333
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "Timeout(5.0)",
+      "got": "DEFAULT_TIMEOUT",
+      "got_raw": "DEFAULT_TIMEOUT",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.07407407407407407
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "Timeout(2.0)",
+      "got": "Timeout(2.0)",
+      "got_raw": "Timeout(2.0)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "Timeout(2.0)",
+      "got": "2.0",
+      "got_raw": "2.0",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "supabase/supabase-py",
+      "expected": "headers.items()",
+      "got": "headers.items()",
+      "got_raw": "headers.items()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "OPTION_SCOPE_FLAG_MAP",
+      "got": "OPTION_SCOPE_FLAG_MAP",
+      "got_raw": "OPTION_SCOPE_FLAG_MAP",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "\"tmuxp\"",
+      "got": "old_test_sessions[0]",
+      "got_raw": "old_test_sessions[0]",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.07407407407407407
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "TEST_SESSION_NAME",
+      "got": "TEST_SESSION_NAME",
+      "got_raw": "TEST_SESSION_NAME",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "version",
+      "got": "version",
+      "got_raw": "version",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "expected_version",
+      "got": "expected_version",
+      "got_raw": "expected_version",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.BadSessionName)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.2686424829558855,
+      "edit_similarity": 0.26666666666666666
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.LibTmuxException)",
+      "got": "exc.LibTmuxException)",
+      "got_raw": "exc.LibTmuxException)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.TmuxCommandNotFound)",
+      "got": "exc.LibTmuxError)",
+      "got_raw": "exc.LibTmuxError)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.4878048780487805
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "3",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "\"test_2\"",
+      "got": "\"test_2\"",
+      "got_raw": "\"test_2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "1",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "50",
+      "got": "100",
+      "got_raw": "100",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "100",
+      "got": "100",
+      "got_raw": "100",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "150",
+      "got": "150",
+      "got_raw": "150",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "[\"\"]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.39640237166757364,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "True",
+      "got": "\"on\"",
+      "got_raw": "\"on\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "4",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "10",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "50",
+      "got": "50",
+      "got_raw": "50",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "75",
+      "got": "75",
+      "got_raw": "75",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "\"$\"",
+      "got": "\"$",
+      "got_raw": "\"$ \"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "\"8\"",
+      "got": "\"10\"",
+      "got_raw": "\"10\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "str(w[0].message)",
+      "got": "str(w[0].message)",
+      "got_raw": "str(w[0].message)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "\"status off\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "server2.socket_name",
+      "got": "server2.socket_name",
+      "got_raw": "server2.socket_name",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "88",
+      "got": "88",
+      "got_raw": "88",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "256",
+      "got": "256",
+      "got_raw": "256",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "cmd",
+      "got": "cmd",
+      "got_raw": "cmd",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "\"32\"",
+      "got": "32",
+      "got_raw": "32",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "\"HI\"",
+      "got": "\"HI\"",
+      "got_raw": "\"HI\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "v",
+      "got": "v",
+      "got_raw": "v",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "20",
+      "got": "20",
+      "got_raw": "20",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "40",
+      "got": "40",
+      "got_raw": "40",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "\"2\"",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "\"1\"",
+      "got": "\"1\"",
+      "got_raw": "\"1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "\"3\"",
+      "got": "\"0\"",
+      "got_raw": "\"0\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "\"4\"",
+      "got": "\"4\"",
+      "got_raw": "\"4\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "pane",
+      "got": "pane",
+      "got_raw": "pane",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "t.cast(\"type[Exception]\", raises))",
+      "got": "ValueError) if raises else does_not_raise()",
+      "got_raw": "ValueError) if raises else does_not_raise()",
+      "exact_match": false,
+      "code_bleu": 0.17924396074436147,
+      "edit_similarity": 0.3116883116883117
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "v",
+      "got": "v",
+      "got_raw": "v",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "20",
+      "got": "20",
+      "got_raw": "20",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "40",
+      "got": "40",
+      "got_raw": "40",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "expected_result",
+      "got": "expected_result",
+      "got_raw": "expected_result",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "version",
+      "got": "version",
+      "got_raw": "version",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "TMUX_NEXT_VERSION",
+      "got": "TMUX_NEXT_VERSION",
+      "got_raw": "TMUX_NEXT_VERSION",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.BadSessionName)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.2686424829558855,
+      "edit_similarity": 0.26666666666666666
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.LibTmuxException)",
+      "got": "exc.TmuxVersionError)",
+      "got_raw": "exc.TmuxVersionError)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.6190476190476191
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.TmuxCommandNotFound)",
+      "got": "exc.LibTmuxError)",
+      "got_raw": "exc.LibTmuxError)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.4878048780487805
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "f\"{TMUX_MAX_VERSION}-master\"",
+      "got": "\"master\"",
+      "got_raw": "\"master\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "f\"{TMUX_MAX_VERSION}-openbsd\"",
+      "got": "\"0.1.0\"",
+      "got_raw": "\"0.1.0\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.1111111111111111
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Pane\\.get\\(\\) was deprecated\")",
+      "got": "exc.DeprecatedError)",
+      "got_raw": "exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.31426968052735443,
+      "edit_similarity": 0.5063291139240507
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Pane\\[key\\] lookup was deprecated\")",
+      "got": "exc.DeprecatedError)",
+      "got_raw": "exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.31426968052735443,
+      "edit_similarity": 0.47619047619047616
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Pane\\.resize_pane\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.39603960396039606
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Pane\\.select_pane\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.39603960396039606
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Pane\\.split_window\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.39215686274509803
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Server\\.children was deprecated\")",
+      "got": "exc.DeprecatedError)",
+      "got_raw": "exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.31426968052735443,
+      "edit_similarity": 0.4878048780487805
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Server\\.where\\(\\) was deprecated\")",
+      "got": "exc.DeprecatedError)",
+      "got_raw": "exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.31426968052735443,
+      "edit_similarity": 0.4819277108433735
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Server\\._sessions was deprecated\")",
+      "got": "exc.DeprecatedError)",
+      "got_raw": "exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.31426968052735443,
+      "edit_similarity": 0.4819277108433735
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Server\\.get_by_id\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.39603960396039606
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Server\\.find_where\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.39215686274509803
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Server\\.kill_server\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.3883495145631068
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Server\\._list_panes\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.3883495145631068
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Server\\.list_sessions\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.38095238095238093
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Server\\._list_windows\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.38095238095238093
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Server\\._update_panes\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.38095238095238093
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Server\\._list_sessions\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.37735849056603776
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Server\\._update_windows\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.37383177570093457
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Session\\.get\\(\\) was deprecated\")",
+      "got": "exc.DeprecatedError)",
+      "got_raw": "exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.31426968052735443,
+      "edit_similarity": 0.4878048780487805
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Session\\.children was deprecated\")",
+      "got": "exc.DeprecatedError)",
+      "got_raw": "exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.31426968052735443,
+      "edit_similarity": 0.4819277108433735
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Session\\._windows was deprecated\")",
+      "got": "exc.DeprecatedError)",
+      "got_raw": "exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.31426968052735443,
+      "edit_similarity": 0.4819277108433735
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Session\\.where\\(\\) was deprecated\")",
+      "got": "exc.DeprecatedError)",
+      "got_raw": "exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.31426968052735443,
+      "edit_similarity": 0.47619047619047616
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Session\\[key\\] lookup was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.39603960396039606
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Session\\.attached_pane was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.39215686274509803
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Session\\.get_by_id\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.39215686274509803
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Session\\.find_where\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.3883495145631068
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Session\\.attached_window was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.38461538461538464
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Session\\.kill_session\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.38095238095238093
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Session\\.list_windows\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.38095238095238093
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Session\\._list_windows\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.37735849056603776
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Session\\.attach_session\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.37383177570093457
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Server\\.where\\(\\) was deprecated\")",
+      "got": "exc.DeprecatedError)",
+      "got_raw": "exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.31426968052735443,
+      "edit_similarity": 0.4819277108433735
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Window\\.where\\(\\) was deprecated\")",
+      "got": "exc.DeprecatedError)",
+      "got_raw": "exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.31426968052735443,
+      "edit_similarity": 0.4819277108433735
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Session\\.where\\(\\) was deprecated\")",
+      "got": "exc.DeprecatedError)",
+      "got_raw": "exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.31426968052735443,
+      "edit_similarity": 0.47619047619047616
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Server\\.get_by_id\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.39603960396039606
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Window\\.get_by_id\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.39603960396039606
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Server\\.find_where\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.39215686274509803
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Window\\.find_where\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.39215686274509803
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Session\\.get_by_id\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.39215686274509803
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Session\\.find_where\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.3883495145631068
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "t.cast(\"type[Exception]\", raises))",
+      "got": "ValueError) if raises else does_not_raise()",
+      "got_raw": "ValueError) if raises else does_not_raise()\n    )",
+      "exact_match": false,
+      "code_bleu": 0.17924396074436147,
+      "edit_similarity": 0.3116883116883117
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Window\\._panes was deprecated\")",
+      "got": "exc.DeprecatedError)",
+      "got_raw": "exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.31426968052735443,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Window\\.get\\(\\) was deprecated\")",
+      "got": "exc.DeprecatedError)",
+      "got_raw": "exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.31426968052735443,
+      "edit_similarity": 0.49382716049382713
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Window\\.children was deprecated\")",
+      "got": "exc.DeprecatedError)",
+      "got_raw": "exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.31426968052735443,
+      "edit_similarity": 0.4878048780487805
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Window\\.where\\(\\) was deprecated\")",
+      "got": "exc.DeprecatedError)",
+      "got_raw": "exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.31426968052735443,
+      "edit_similarity": 0.4819277108433735
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Window\\[key\\] lookup was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Window\\.attached_pane was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.39603960396039606
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Window\\.get_by_id\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.39603960396039606
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Window\\.find_where\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.39215686274509803
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Window\\.list_panes\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.39215686274509803
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Window\\.kill_window\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.3883495145631068
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Window\\._list_panes\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.3883495145631068
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Window\\.split_window\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.38461538461538464
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.DeprecatedError, match=r\"Window\\.select_window\\(\\) was deprecated\")",
+      "got": "pytest.raises(exc.DeprecatedError)",
+      "got_raw": " pytest.raises(exc.DeprecatedError)",
+      "exact_match": false,
+      "code_bleu": 0.28053195903226175,
+      "edit_similarity": 0.38095238095238093
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "8",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "10",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "0.1",
+      "got": "0.1",
+      "got_raw": "0.1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "0.05",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "\"libtmux_\"",
+      "got": "\"test-session-\"",
+      "got_raw": "\"test-session-\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.24
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "\"value1\"",
+      "got": "\"value1\"",
+      "got_raw": "\"value1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "\"value2\"",
+      "got": "\"value2\"",
+      "got_raw": "\"value2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "os.environ",
+      "got": "os.environ",
+      "got_raw": "os.environ",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "\"new_value\"",
+      "got": "\"new_value\"",
+      "got_raw": "\"new_value\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "\"new_value1\"",
+      "got": "\"new_value1\"",
+      "got_raw": "\"new_value1\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "\"original_value\"",
+      "got": "\"original_value\"",
+      "got_raw": "\"original_value\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "8",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "rng",
+      "got": "rng",
+      "got_raw": "rng",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "result2",
+      "got": "result2",
+      "got_raw": "result2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "third_name",
+      "got": "third_name",
+      "got_raw": "third_name",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "end - ini",
+      "got": "end - ini)",
+      "got_raw": "end - ini)",
+      "exact_match": false,
+      "code_bleu": 0.668740304976422,
+      "edit_similarity": 0.9473684210526315
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "exc.WaitTimeout)",
+      "got": "exc.TimeoutError)",
+      "got_raw": "exc.TimeoutError)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.7272727272727273
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "expected_value",
+      "got": "expected_value",
+      "got_raw": "expected_value",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "test_case.expected",
+      "got": "test_case.expected",
+      "got_raw": "test_case.expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "test_case.expected_keys",
+      "got": "sorted(test_case.expected_keys)",
+      "got_raw": "sorted(test_case.expected_keys)",
+      "exact_match": false,
+      "code_bleu": 0.4111336169005197,
+      "edit_similarity": 0.8518518518518519
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "test_case.expected_values",
+      "got": "test_case.values",
+      "got_raw": "test_case.values",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.7804878048780488
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "window_name",
+      "got": "window_name",
+      "got_raw": "window_name",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "session_name",
+      "got": "session_name",
+      "got_raw": "session_name",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "RuntimeError)",
+      "got": "RuntimeError)",
+      "got_raw": "RuntimeError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "tmux-python/libtmux",
+      "expected": "initial_windows",
+      "got": "initial_windows",
+      "got_raw": "initial_windows",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "6",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "1",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "RDSError)",
+      "got": "RDSError)",
+      "got_raw": "RDSError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "str(error)",
+      "got": "str(error)",
+      "got_raw": "str(error)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "[ call(ANY), call(ANY), ]",
+      "got": "[call(\"test\")]",
+      "got_raw": "[call(\"test\")]",
+      "exact_match": false,
+      "code_bleu": 0.22749707020240187,
+      "edit_similarity": 0.41025641025641024
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"book\"",
+      "got": "\"book\"",
+      "got_raw": "\"book\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "tables",
+      "got": "tables",
+      "got_raw": "tables",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "[\"id\"]",
+      "got": "\"id\"",
+      "got_raw": "\"id\"",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"city\"",
+      "got": "\"continent\"",
+      "got_raw": "\"continent\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.47058823529411764
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"123\"",
+      "got": "\"123\"",
+      "got_raw": "\"123\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"test\"",
+      "got": "\"test\"",
+      "got_raw": "\"test\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"value\"",
+      "got": "\"value\"",
+      "got_raw": "\"value\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "5",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "6",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "2002",
+      "got": "2002",
+      "got_raw": "2002",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "[1, 2]",
+      "got": "[1, 2]",
+      "got_raw": "[1, 2]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "[3, 4, 5]",
+      "got": "[3, 4, 5]",
+      "got_raw": "[3, 4, 5]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "{2001, 2003}",
+      "got": "{2002, 2003}",
+      "got_raw": "{2002, 2003}",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.9166666666666666
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"pop size: 2\")",
+      "got": "call(\"Popping from redis queue\"))",
+      "got_raw": "call(\"Popping from redis queue\"))",
+      "exact_match": false,
+      "code_bleu": 0.2730120862709067,
+      "edit_similarity": 0.2978723404255319
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"pop size: 3\")",
+      "got": "\"pop size: 0\")",
+      "got_raw": "\"pop size: 0\")",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.9285714285714286
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"foo:something\"",
+      "got": "\"foo:something\"",
+      "got_raw": "\"foo:something\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "ConnectionError)",
+      "got": "ConnectionError)",
+      "got_raw": "ConnectionError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "{3001, 3002, 3003}",
+      "got": "{3001, 3002, 3003}",
+      "got_raw": "{3001, 3002, 3003}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "10",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "100",
+      "got": "100",
+      "got_raw": "100",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "2.0",
+      "got": "2.0",
+      "got_raw": "2.0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "url)",
+      "got": "_search_client.assert_called_once()",
+      "got_raw": "_search_client.assert_called_once()",
+      "exact_match": false,
+      "code_bleu": 0.2730120862709067,
+      "edit_similarity": 0.15384615384615385
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "60.0",
+      "got": "60.0",
+      "got_raw": "60.0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "Exception)",
+      "got": "Exception)",
+      "got_raw": "Exception)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"OpenSearch\"",
+      "got": "\"opensearch\"",
+      "got_raw": "\"opensearch\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8333333333333334
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "RuntimeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "str(excinfo.value)",
+      "got": "str(excinfo.value)",
+      "got_raw": "str(excinfo.value)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "url",
+      "got": "url",
+      "got_raw": "url",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"http://some-domain:33\"",
+      "got": "\"http://some-domain:33\"",
+      "got_raw": "\"http://some-domain:33\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "4",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "3",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "[]",
+      "got": "[doc]",
+      "got_raw": "[doc]",
+      "exact_match": false,
+      "code_bleu": 0.49999999999999994,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "val",
+      "got": "val",
+      "got_raw": "val",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "nodes)",
+      "got": "data)",
+      "got_raw": "data)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "nodes)",
+      "got": "data)",
+      "got_raw": "data)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "docs[0][\"_id\"]",
+      "got": "docs[0][\"title\"]",
+      "got_raw": "docs[0][\"title\"]",
+      "exact_match": false,
+      "code_bleu": 0.7071067811865475,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "docs[1][\"_id\"]",
+      "got": "docs[1][\"_id\"]",
+      "got_raw": "docs[1][\"_id\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "docs[2][\"_id\"]",
+      "got": "docs[2][\"_id\"]",
+      "got_raw": "docs[2][\"_id\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "docs[0][\"_source\"]",
+      "got": "docs[0]",
+      "got_raw": "docs[0]",
+      "exact_match": false,
+      "code_bleu": 0.5714285714285714,
+      "edit_similarity": 0.56
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "str(excinfo.value)",
+      "got": "str(excinfo.value)",
+      "got_raw": "str(excinfo.value)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "NodeAttributeError)",
+      "got": "NodeAttributeError)",
+      "got_raw": "NodeAttributeError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "ColumnNotFoundError)",
+      "got": "ColumnNotFoundError)",
+      "got_raw": "ColumnNotFoundError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "doc.get(\"node\", {}))",
+      "got": "\"testdb\")",
+      "got_raw": "\"testdb\")",
+      "exact_match": false,
+      "code_bleu": 0.1074569931823542,
+      "edit_similarity": 0.27586206896551724
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"abc\"",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"def\"",
+      "got": "\"def\"",
+      "got_raw": "\"def\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"ghi\"",
+      "got": "\"ghi\"",
+      "got_raw": "\"ghi\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "nodes)",
+      "got": "data)",
+      "got_raw": "data)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"abc\"",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"def\"",
+      "got": "\"def\"",
+      "got_raw": "\"def\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"ghi\"",
+      "got": "\"ghi\"",
+      "got_raw": "\"ghi\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "nodes)",
+      "got": "data)",
+      "got_raw": "data)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.36363636363636365
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "123",
+      "got": "123",
+      "got_raw": "123",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "{\"id\": \"my_id\"}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.23784142300054423,
+      "edit_similarity": 0.23529411764705882
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "{ \"id\": 1, \"code\": \"ABC\", }",
+      "got": "{\"id\": 1, \"code\": \"\"}",
+      "got_raw": "{\"id\": 1, \"code\": \"\"}",
+      "exact_match": false,
+      "code_bleu": 0.7042880610329794,
+      "edit_similarity": 0.875
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "{ \"id\": 1, \"my_code\": None, }",
+      "got": "{\"id\": 1, \"my_code\": None}",
+      "got_raw": "{\"id\": 1, \"my_code\": None}",
+      "exact_match": false,
+      "code_bleu": 0.8132418032488603,
+      "edit_similarity": 0.9454545454545454
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "{ \"id\": 1, \"code\": \"ABCDEF\", }",
+      "got": "row",
+      "got_raw": "row",
+      "exact_match": false,
+      "code_bleu": 0.05,
+      "edit_similarity": 0.06060606060606061
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "{ \"user\": {\"id\": [1, 2, 3]}, }",
+      "got": "[\"id\"]",
+      "got_raw": "[\"id\"]",
+      "exact_match": false,
+      "code_bleu": 0.10074093110845705,
+      "edit_similarity": 0.2777777777777778
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "{ \"child_table\": {\"name\": \"child_name\"}, }",
+      "got": "{ \"table\": \"child_table\", \"columns\": [\"name\"], \"relationship\": { \"variant\": \"object\", \"type\": \"one_to_one\", }, \"transform\": { \"rename\": { \"name\": \"child_name\", }, }, }",
+      "got_raw": "{ \"table\": \"child_table\", \"columns\": [\"name\"], \"relationship\": { \"variant\": \"object\", \"type\": \"one_to_one\", }, \"transform\": { \"rename\": { \"name\": \"child_name\", }, }, }",
+      "exact_match": false,
+      "code_bleu": 0.15148443669209727,
+      "edit_similarity": 0.4019138755980861
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "{ \"id\": 1, \"codes\": [\"A=B\", \"C=D\", \"E=F\"], }",
+      "got": "{\"id\": 1, \"codes\": [\"A=\", \"C=\", \"E=\"]}",
+      "got_raw": "{\"id\": 1, \"codes\": [\"A=\", \"C=\", \"E=\"]}",
+      "exact_match": false,
+      "code_bleu": 0.5271949923659522,
+      "edit_similarity": 0.926829268292683
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "{ \"first\": \"A\", \"second\": \"B\", \"combined\": \"AB\", }",
+      "got": "{\"combined\": \"A,B\"}",
+      "got_raw": "{\"combined\": \"A,B\"}",
+      "exact_match": false,
+      "code_bleu": 0.1458029608799511,
+      "edit_similarity": 0.5217391304347826
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "{ \"code\": {\"@\": \"",
+      "got": "{ \"code\": {\"@\": \"",
+      "got_raw": "{ \"code\": {\"@\": \"#\"}, }",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "{ \"id\": 1, \"code\": \"ABC=DEF GHI\", \"name\": \"unchanged\", }",
+      "got": "{\"id\": 1, \"code\": \"ABC DEF_GHI\", \"name\": \"unchanged\"}",
+      "got_raw": "{\"id\": 1, \"code\": \"ABC DEF_GHI\", \"name\": \"unchanged\"}",
+      "exact_match": false,
+      "code_bleu": 0.6629032286663868,
+      "edit_similarity": 0.9357798165137615
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "{ \"id\": 1, \"first\": \"A\", \"combined\": \"A-second-third\", }",
+      "got": "{\"id\": 1, \"first\": \"A\", \"second\": None, \"third\": None}",
+      "got_raw": "{\"id\": 1, \"first\": \"A\", \"second\": None, \"third\": None}",
+      "exact_match": false,
+      "code_bleu": 0.5465253425714404,
+      "edit_similarity": 0.7090909090909091
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "primary_keys",
+      "got": "primary_keys",
+      "got_raw": "primary_keys",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "foreign_keys",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "sorted(foreign_keys)",
+      "got": "foreign_keys",
+      "got_raw": "foreign_keys",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "nodes)",
+      "got": "sync.sync())",
+      "got_raw": "sync.sync())",
+      "exact_match": false,
+      "code_bleu": 0.22089591134157885,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "[ { \"_id\": \"abc\", \"_index\": \"testdb\", \"_source\": { \"_meta\": { \"book\": {\"isbn\": [\"abc\"]}, \"contact\": {\"id\": [1, 2]}, \"user\": {\"id\": [1, 2]}, }, \"buyer\": { \"contacts\": [ { \"contact_items\": None, \"id\": 1, \"name\": \"Contact 1\", } ], \"id\": 1, \"name\": \"Fonzy Bear\", }, \"description\": \"Tigers are fierce creatures\", \"isbn\": \"abc\", \"seller\": { \"contacts\": [ { \"contact_items\": None, \"id\": 2, \"name\": \"Contact 2\", } ], \"id\": 2, \"name\": \"Jack Jones\", }, \"title\": \"The Tiger Club\", }, } ]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.009009144810626675,
+      "edit_similarity": 0.0041841004184100415
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "3",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "ValueError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "str(excinfo.value)",
+      "got": "\"Missing",
+      "got_raw": "\"Missing config\"",
+      "exact_match": false,
+      "code_bleu": 0.15060033393497413,
+      "edit_similarity": 0.23076923076923078
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "FileNotFoundError)",
+      "got": "FileNotFoundError)",
+      "got_raw": "FileNotFoundError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"http://localhost:9999\"",
+      "got": "f\"http://kermit:1234@localhost:{ELASTICSEARCH_PORT}\"",
+      "got_raw": "f\"http://kermit:1234@localhost:{ELASTICSEARCH_PORT}\"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.5066666666666667
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"redis://skynet:6379/0\"",
+      "got": "\"redis://skynet:6379/0\"",
+      "got_raw": "\"redis://skynet:6379/0\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"redis://localhost:6379/0\"",
+      "got": "\"redis://localhost:6379/0\"",
+      "got_raw": "\"redis://localhost:6379/0\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"redis://:1234@localhost:9999/0\"",
+      "got": "\"redis://localhost:6379/0\"",
+      "got_raw": "\"redis://localhost:6379/0\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7931034482758621
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"redis://kermit:1234@localhost:6379/0\"",
+      "got": "\"redis://user:password@localhost:6379/0\"",
+      "got_raw": "\"redis://user:password@localhost:6379/0\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7692307692307693
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"redis://kermit:1234@localhost:9999/0\"",
+      "got": "\"kermit:1234\"",
+      "got_raw": "\"kermit:1234\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5098039215686274
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "f\"http://localhost:{ELASTICSEARCH_PORT}\"",
+      "got": "\"Invalid",
+      "got_raw": "\"Invalid port\"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.125
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"Connecting to Redis without password.\")",
+      "got": "_logger.error.assert_called_with(\"Redis connection failed\")",
+      "got_raw": "_logger.error.assert_called_with(\"Redis connection failed\")",
+      "exact_match": false,
+      "code_bleu": 0.16036590969929357,
+      "edit_similarity": 0.2828282828282828
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "f\"https://localhost:{ELASTICSEARCH_PORT}\"",
+      "got": "\"postgres://postgres:postgres@localhost:5432/postgres\"",
+      "got_raw": "\"postgres://postgres:postgres@localhost:5432/postgres\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.35789473684210527
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"mysql+pymysql://kermit:12345@localhost:3306/mydb\"",
+      "got": "\"mysql\"",
+      "got_raw": "\"mysql\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.24561403508771928
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "f\"http://kermit:1234@localhost:{ELASTICSEARCH_PORT}\"",
+      "got": "\"Using",
+      "got_raw": "\"Using https for search\"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.06896551724137931
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"postgresql+psycopg2://kermit:12345@localhost:5432/mydb\"",
+      "got": "\"postgres\"",
+      "got_raw": "\"postgres\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.30303030303030304
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"Connecting to Redis with custom username and password.\")",
+      "got": "_logger.error.assert_called_with(\"Invalid username and password\")",
+      "got_raw": "_logger.error.assert_called_with(\"Invalid username and password\")",
+      "exact_match": false,
+      "code_bleu": 0.16036590969929357,
+      "edit_similarity": 0.5409836065573771
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "str( excinfo.value )",
+      "got": "str(excinfo.value)",
+      "got_raw": "str(excinfo.value)",
+      "exact_match": false,
+      "code_bleu": 1.0,
+      "edit_similarity": 0.9473684210526315
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "5",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "3",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "15",
+      "got": "15",
+      "got_raw": "15",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "url",
+      "got": "url",
+      "got_raw": "url",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "\"0\"",
+      "got": "\"0\"",
+      "got_raw": "\"0\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "ddl",
+      "got": "ddl",
+      "got_raw": "ddl",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "toluaina/pgsync",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "0.5",
+      "got": "0.5",
+      "got_raw": "0.5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "0.3",
+      "got": "0.3",
+      "got_raw": "0.3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"run_sql\"",
+      "got": "\"run_sql\"",
+      "got_raw": "\"run_sql\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"Show me top customers\"",
+      "got": "\"Show me top customers\"",
+      "got_raw": "\"Show me top customers\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "100",
+      "got": "100",
+      "got_raw": "100",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "0.95",
+      "got": "0.95",
+      "got_raw": "0.95",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "0.88",
+      "got": "0.88",
+      "got_raw": "0.88",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "2",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "0.8",
+      "got": "0.8",
+      "got_raw": "0.8",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"auto\"",
+      "got": "\"test_tool\"",
+      "got_raw": "\"test_tool\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.47058823529411764
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"gpt-5\"",
+      "got": "\"gpt-5\"",
+      "got_raw": "\"gpt-5\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "payload",
+      "got": "payload",
+      "got_raw": "payload",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"gpt-4o\"",
+      "got": "\"gpt-4o\"",
+      "got_raw": "\"gpt-4o\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"system\"",
+      "got": "\"user\"",
+      "got_raw": "\"user\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"env-key\"",
+      "got": "\"env-key\"",
+      "got_raw": "\"env-key\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"test-key\"",
+      "got": "\"test-key\"",
+      "got_raw": "\"test-key\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"function\"",
+      "got": "\"object\"",
+      "got_raw": "\"object\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "call_kwargs",
+      "got": "call_kwargs",
+      "got_raw": "call_kwargs",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"test_tool\"",
+      "got": "\"test_tool\"",
+      "got_raw": "\"test_tool\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"2024-10-21\"",
+      "got": "\"2024-10-21\"",
+      "got_raw": "\"2024-10-21\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"2024-06-01\"",
+      "got": "\"2024-06-01\"",
+      "got_raw": "\"2024-06-01\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"new_collection\"",
+      "got": "\"new_collection\"",
+      "got_raw": "\"new_collection\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"test_collection\"",
+      "got": "\"test_collection\"",
+      "got_raw": "\"test_collection\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "[\"test question 1\", \"test question 2\"]",
+      "got": "[\"test question 1\", \"test question 2\"]",
+      "got_raw": "[\"test question 1\", \"test question 2\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "0.7",
+      "got": "0.7",
+      "got_raw": "0.7",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"gemini-2.5-pro\"",
+      "got": "\"gemini-2.5-pro\"",
+      "got_raw": "\"gemini-2.5-pro\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "response.content",
+      "got": "response.content",
+      "got_raw": "response.content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "ValueError, match=\"Google API key is required\")",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.3793103448275862
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"What are the total sales?\"",
+      "got": "\"What are the total sales?\"",
+      "got_raw": "\"What are the total sales?\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"SELECT SUM(Total) FROM Invoice\"",
+      "got": "\"SELECT SUM(Total) FROM Invoice\"",
+      "got_raw": "\"SELECT SUM(Total) FROM Invoice\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"The Invoice table contains all sales transactions.\"",
+      "got": "\"Content of the text memory\"",
+      "got_raw": "\"Content of the text memory\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "call",
+      "got": "call",
+      "got_raw": "call",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"test_user\"",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "first_request.system_prompt",
+      "got": "first_request.system_prompt",
+      "got_raw": "first_request.system_prompt",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "content",
+      "got": "content",
+      "got_raw": "content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"success\"",
+      "got": "\"Success\"",
+      "got_raw": "\"Success\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "content.lower()",
+      "got": "content",
+      "got_raw": "content",
+      "exact_match": false,
+      "code_bleu": 0.11892071150027211,
+      "edit_similarity": 0.6363636363636364
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "ComponentType.CARD",
+      "got": "ComponentType.CARD",
+      "got_raw": "ComponentType.CARD",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "status.message.lower()",
+      "got": "status.message",
+      "got_raw": "status.message",
+      "exact_match": false,
+      "code_bleu": 0.36038417796587763,
+      "edit_similarity": 0.7777777777777778
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "user_result.result_for_llm",
+      "got": "user_result.result_for_llm",
+      "got_raw": "user_result.result_for_llm",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "0.5",
+      "got": "0.5",
+      "got_raw": "0.5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "4096",
+      "got": "4096",
+      "got_raw": "4096",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"user\"",
+      "got": "\"user\"",
+      "got_raw": "\"user\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"llama3.2\"",
+      "got": "\"llama3.2\"",
+      "got_raw": "\"llama3.2\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "response.content",
+      "got": "response.content",
+      "got_raw": "response.content",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "\"http://localhost:11434\"",
+      "got": "\"http://localhost:11434\"",
+      "got_raw": "\"http://localhost:11434\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "vanna-ai/vanna",
+      "expected": "4",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "\"new.value\"",
+      "got": "\"new.value\"",
+      "got_raw": "\"new.value\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "AttributeError)",
+      "got": "AttributeError)",
+      "got_raw": "AttributeError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "DEFAULTS[\"INIT_API_FN\"]",
+      "got": "DEFAULTS[\"INIT_API_FN\"]",
+      "got_raw": "DEFAULTS[\"INIT_API_FN\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "\"custom.module.init_api\"",
+      "got": "\"custom.module.init_api\"",
+      "got_raw": "\"custom.module.init_api\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "VERSION",
+      "got": "VERSION",
+      "got_raw": "VERSION",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "thread.id",
+      "got": "thread.id",
+      "got_raw": "thread.id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "\"New name\"",
+      "got": "\"New name\"",
+      "got_raw": "\"New name\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "PACKAGE_NAME",
+      "got": "PACKAGE_NAME",
+      "got_raw": "PACKAGE_NAME",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "assistant_id",
+      "got": "assistant_id",
+      "got_raw": "assistant_id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "30",
+      "got": "30",
+      "got_raw": "30",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "2001",
+      "got": "2001",
+      "got_raw": "2001",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "False",
+      "got": "False",
+      "got_raw": "False",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "\"John\"",
+      "got": "\"John\"",
+      "got_raw": "\"John\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "\"Shrek\"",
+      "got": "\"Shrek\"",
+      "got_raw": "\"Shrek\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "\"system\"",
+      "got": "\"text\"",
+      "got_raw": "\"text\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "call_kwargs",
+      "got": "call_kwargs",
+      "got_raw": "call_kwargs",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "[\"Animation\", \"Comedy\"]",
+      "got": "[\"Animation\", \"Comedy\"]",
+      "got_raw": "[\"Animation\", \"Comedy\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "[m[\"data\"][\"id\"] for m in expected_messages]",
+      "got": "[m[\"data\"][\"id\"] for m in expected_messages]",
+      "got_raw": "[m[\"data\"][\"id\"] for m in expected_messages]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "\"You are a temperature bot. Today is 2024-06-09.\"",
+      "got": "\"Today is 2024-06-09.\"",
+      "got_raw": "\"Today is 2024-06-09.\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6197183098591549
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "\"11 W 53rd St, New York, NY 10019, United States.\"",
+      "got": "\"11 W 53rd St, New York, NY 10019, United States.\"",
+      "got_raw": "\"11 W 53rd St, New York, NY 10019, United States.\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "{\"input\": \"What is the temperature today in Recife?\"})",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "\"I'm at Central Park W & 79st, New York, NY 10024, United States.\"",
+      "got": "\"11 W 53rd St, New York, NY 10019, United States\"",
+      "got_raw": "\"11 W 53rd St, New York, NY 10019, United States\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6956521739130435
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "\"You're right by the Museum of Modern Art (MoMA), which houses an impressive collection of modern and contemporary art, including works by Van Gogh and Warhol. Enjoy exploring the vibrant gallery spaces and temporary exhibitions. It's a must-visit for art enthusiasts!\"",
+      "got": "\"Tour Guide Assistant\"",
+      "got_raw": "\"Tour Guide Assistant\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.027491408934707903
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "\"You're right next to the American Museum of Natural History, where you can explore fascinating exhibits about dinosaurs, space, and human cultures. Additionally, enjoy a stroll through Central Park to experience its beautiful landscapes and iconic landmarks like Belvedere Castle and Bow Bridge. Don't miss the opportunity to visit both these highlights!\"",
+      "got": "\"Tour Guide Assistant\"",
+      "got_raw": "\"Tour Guide Assistant\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.026455026455026454
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "[ \"tool_d\", \"tool_c\", \"tool_b\", \"tool_a\", ]",
+      "got": "[\"tool_a\", \"tool_b\", \"tool_c\", \"tool_d\"]",
+      "got_raw": "[\"tool_a\", \"tool_b\", \"tool_c\", \"tool_d\"]",
+      "exact_match": false,
+      "code_bleu": 0.39124878475049224,
+      "edit_similarity": 0.7228915662650602
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "[ m[\"data\"].get(attr) for m in expected_messages ]",
+      "got": "[None]",
+      "got_raw": "[None]",
+      "exact_match": false,
+      "code_bleu": 0.09999999999999999,
+      "edit_similarity": 0.17857142857142858
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "\"Hello\"",
+      "got": "\"Hello\"",
+      "got_raw": "\"Hello\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "\"\"",
+      "got": "\"assistant\"",
+      "got_raw": "\"assistant\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "user",
+      "got": "user",
+      "got_raw": "user",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "thread",
+      "got": "thread",
+      "got_raw": "thread",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "\"My thread\"",
+      "got": "\"My thread\"",
+      "got_raw": "\"My thread\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "assistant_id",
+      "got": "assistant_id",
+      "got_raw": "assistant_id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "\"My updated thread\"",
+      "got": "\"My updated thread\"",
+      "got_raw": "\"My updated thread\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "vintasoftware/django-ai-assistant",
+      "expected": "AIUserNotAllowedError)",
+      "got": "AIUserNotAllowedError)",
+      "got_raw": "AIUserNotAllowedError)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "px_terminal.red(\" apa \") + px_terminal.yellow(\" b\") + px_terminal.blue(\" \") + px_terminal.inverse_video(\" \")",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "px_terminal.red(\" \u00e5pa \") + px_terminal.yellow(\" b\") + px_terminal.blue(\" \") + px_terminal.inverse_video(\" \")",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "5",
+      "got": "\"/a",
+      "got_raw": "\"/a b c\"",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"\ud83d\ude00\"",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"sh\"",
+      "got": "\"sh\"",
+      "got_raw": "\"sh\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"go\"",
+      "got": "\"go",
+      "got_raw": "\"go --version\"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"hej\"",
+      "got": "\"hej\"",
+      "got_raw": "\"hej\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"mod\"",
+      "got": "\"mod\"",
+      "got_raw": "\"mod\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "1)",
+      "got": "1)",
+      "got_raw": "1)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "4)",
+      "got": "4)",
+      "got_raw": "4)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "8)",
+      "got": "8)",
+      "got_raw": "8)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "96)",
+      "got": "96)",
+      "got_raw": "96)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "(1, 2)",
+      "got": "(1, 2)",
+      "got_raw": "(1, 2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "physical",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "(12, 34)",
+      "got": "(12, 34)",
+      "got_raw": "(12, 34)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "(None, None)",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.18072040072196896,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "expected_logical",
+      "got": "expected_logical",
+      "got_raw": "expected_logical",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "expected_physical",
+      "got": "expected_physical",
+      "got_raw": "expected_physical",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"/\"",
+      "got": "\"/\"",
+      "got_raw": "\"/\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[friend]",
+      "got": "[234]",
+      "got_raw": "[234]",
+      "exact_match": false,
+      "code_bleu": 0.49999999999999994,
+      "edit_similarity": 0.3076923076923077
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[p1, p2]",
+      "got": "[p1, p2]",
+      "got_raw": "[p1, p2]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"/notroot\"",
+      "got": "\"/notroot\"",
+      "got_raw": "\"/notroot\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[\"a\", \"b\"]",
+      "got": "[\"a\", \"b\"]",
+      "got_raw": "[\"a\", \"b\"]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[\"a\", \"-b\", \"c\"]",
+      "got": "[\"a\", \"c\"]",
+      "got_raw": "[\"a\", \"c\"]",
+      "exact_match": false,
+      "code_bleu": 0.5050762722761053,
+      "edit_similarity": 0.7692307692307693
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"\"",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "123",
+      "got": "123",
+      "got_raw": "123",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"/\"",
+      "got": "\"a",
+      "got_raw": "\"a tDIR n/\"",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"r\"",
+      "got": "\"ar\"",
+      "got_raw": "\"ar\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8571428571428571
+    },
+    {
+      "repo": "walles/px",
+      "expected": "100",
+      "got": "100",
+      "got_raw": "100",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "None",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"localhost:ipp\"",
+      "got": "\"localhost:19091\"",
+      "got_raw": "\"localhost:19091\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"localhost:63342\"",
+      "got": "\"localhost:63342\"",
+      "got_raw": "\"localhost:63342\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "2",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "4",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "{25}",
+      "got": "[25]",
+      "got_raw": "[25]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "walles/px",
+      "expected": "set()",
+      "got": "set()",
+      "got_raw": "set()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "{200}",
+      "got": "[200",
+      "got_raw": "[200, 300]",
+      "exact_match": false,
+      "code_bleu": 0.35818997727451396,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "walles/px",
+      "expected": "{100}",
+      "got": "{100}",
+      "got_raw": "{100}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"Johan\"",
+      "got": "\"Johan\"",
+      "got_raw": "\"Johan\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "{25, 26}",
+      "got": "[25, 26]",
+      "got_raw": "[25, 26]",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"PID 45\"",
+      "got": "\"45\"",
+      "got_raw": "\"45\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "walles/px",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[l2, l1]",
+      "got": "[",
+      "got_raw": "[",
+      "exact_match": false,
+      "code_bleu": 0.11892071150027211,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[(\"a\", 5)]",
+      "got": "[ (\"a\", 0), ]",
+      "got_raw": "[ (\"a\", 0), ]",
+      "exact_match": false,
+      "code_bleu": 0.48549177170732344,
+      "edit_similarity": 0.782608695652174
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[(\"a\", 0), (\"b\", 6), (\"c\", 5)]",
+      "got": "[(\"a\", 0), (\"b\", 4)]",
+      "got_raw": "[(\"a\", 0), (\"b\", 4)]",
+      "exact_match": false,
+      "code_bleu": 0.5535993289690987,
+      "edit_similarity": 0.76
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[process_other_pid, process_other_starttime]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.23784142300054423,
+      "edit_similarity": 0.08695652173913043
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[ (\"a\", 0), (\"b\", 0), (\"c\", 5), ]",
+      "got": "[(0, \"a\"), (1, \"b\"), (2, \"c\")]",
+      "got_raw": "[(0, \"a\"), (1, \"b\"), (2, \"c\")]",
+      "exact_match": false,
+      "code_bleu": 0.21826474986716862,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[ \"init -> \" + px_terminal.bold(\"iTerm\") + \"(1) -> \" + px_terminal.bold(\"fish\") + \"(1)\" ]",
+      "got": "[\"init\"",
+      "got_raw": "[\"init\", \"iTerm\", \"fish\"]",
+      "exact_match": false,
+      "code_bleu": 0.051169996753502,
+      "edit_similarity": 0.14583333333333334
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[ \"init -> \" + px_terminal.bold(\"iTerm\") + \"(3) -> \" + px_terminal.bold(\"fish\") + \"(2)\" ]",
+      "got": "[ \"iTerm\"",
+      "got_raw": "[ \"iTerm\", \"  2 init\", \"  1 init\", \"  1 init\", \"  1 init\", \"  1 init\", \"  1 fish\", \"  1 fish\", \"  1 fish\", \"  1 fish\", \"  1 fish\", \"  1 fish\", \"  1 fish\", \"  1 fish\", \"  1 fish\", \"  1 fish\", \"  1 fish\", \"  1 fish\", \"  1 fish\", \"  1 fish\", \"  1 fish\", \"  ",
+      "exact_match": false,
+      "code_bleu": 0.056628910238224806,
+      "edit_similarity": 0.1836734693877551
+    },
+    {
+      "repo": "walles/px",
+      "expected": "{ \"init -> \" + px_terminal.bold(\"iTerm\") + \"(3)\", \"init -> \" + px_terminal.bold(\"(i)Term\") + \"(1)\", \"init -> \" + px_terminal.bold(\"i(Term)\") + \"(1)\", \"init -> \" + px_terminal.bold(\"iTerm()\") + \"(1)\", }",
+      "got": "{ \"init\", \"init\", \"init\", \"init\", \"init\", \"init\", }",
+      "got_raw": "{ \"init\", \"init\", \"init\", \"init\", \"init\", \"init\", }",
+      "exact_match": false,
+      "code_bleu": 0.043758884217659715,
+      "edit_similarity": 0.0873015873015873
+    },
+    {
+      "repo": "walles/px",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "8",
+      "got": "15",
+      "got_raw": "15",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "(0, 0, 0, 1.0)",
+      "got": "(0.0, 0.0, 0.0, 0.0)",
+      "got_raw": "(0.0, 0.0, 0.0, 0.0)",
+      "exact_match": false,
+      "code_bleu": 0.18575057999133596,
+      "edit_similarity": 0.6470588235294118
+    },
+    {
+      "repo": "walles/px",
+      "expected": "(0, 1, 3, 1.0)",
+      "got": "(1.0 / 6.0, 1.0 / 6.0 + 0.01, 1.0, 1.0)",
+      "got_raw": "(1.0 / 6.0, 1.0 / 6.0 + 0.01, 1.0, 1.0)",
+      "exact_match": false,
+      "code_bleu": 0.17609282679116184,
+      "edit_similarity": 0.49056603773584906
+    },
+    {
+      "repo": "walles/px",
+      "expected": "(2, 3, 3, 1.0)",
+      "got": "(3, 5, 6, 1.0)",
+      "got_raw": "(3, 5, 6, 1.0)",
+      "exact_match": false,
+      "code_bleu": 0.3356891925037239,
+      "edit_similarity": 0.7142857142857143
+    },
+    {
+      "repo": "walles/px",
+      "expected": "px_load.get_load_string((0.3, 0.2, 0.1))",
+      "got": "px_load.get_load_string()",
+      "got_raw": "px_load.get_load_string()",
+      "exact_match": false,
+      "code_bleu": 0.3313628036529378,
+      "edit_similarity": 0.7692307692307693
+    },
+    {
+      "repo": "walles/px",
+      "expected": "px_load.get_load_string((3.0, 0.2, 0.1))",
+      "got": "px_load.get_load_string((0.0, 0.0, 0.0))",
+      "got_raw": "px_load.get_load_string((0.0, 0.0, 0.0))",
+      "exact_match": false,
+      "code_bleu": 0.45258880078905583,
+      "edit_similarity": 0.925
+    },
+    {
+      "repo": "walles/px",
+      "expected": "px_load.get_load_string((2.0, 3.0, 4.0))",
+      "got": "px_load.get_load_string((2.0, 0.2, 0.1))",
+      "got_raw": "px_load.get_load_string((2.0, 0.2, 0.1))",
+      "exact_match": false,
+      "code_bleu": 0.6262844962765468,
+      "edit_similarity": 0.9
+    },
+    {
+      "repo": "walles/px",
+      "expected": "px_load.get_load_string((1.135135, 0.2, 0.1))",
+      "got": "px_load.get_load_string((1.1, 0.2, 0.1))",
+      "got_raw": "px_load.get_load_string((1.1, 0.2, 0.1))",
+      "exact_match": false,
+      "code_bleu": 0.7611606003349892,
+      "edit_similarity": 0.9411764705882353
+    },
+    {
+      "repo": "walles/px",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "caplog.text",
+      "got": "caplog.text",
+      "got_raw": "caplog.text",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "get_users_at(lastline, now, testtime)",
+      "got": "get_users_at(lastline, now, testtime)",
+      "got_raw": "get_users_at(lastline, now, testtime)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "get_users_at(lastline, now, datetime.datetime.now(TIMEZONE))",
+      "got": "get_users_at(lastline, now, now)",
+      "got_raw": "get_users_at(lastline, now, now)",
+      "exact_match": false,
+      "code_bleu": 0.41229609370343756,
+      "edit_similarity": 0.6956521739130435
+    },
+    {
+      "repo": "walles/px",
+      "expected": "datetime.timedelta( 0, hours=1, minutes=29 )",
+      "got": "datetime.timedelta(minutes=1, seconds=29)",
+      "got_raw": "datetime.timedelta(minutes=1, seconds=29)",
+      "exact_match": false,
+      "code_bleu": 0.42931872247894176,
+      "edit_similarity": 0.7058823529411765
+    },
+    {
+      "repo": "walles/px",
+      "expected": "datetime.timedelta( 4, hours=1, minutes=29 )",
+      "got": "datetime.timedelta(4, 0, 1, 29)",
+      "got_raw": "datetime.timedelta(4, 0, 1, 29)",
+      "exact_match": false,
+      "code_bleu": 0.4850673648559458,
+      "edit_similarity": 0.7466666666666667
+    },
+    {
+      "repo": "walles/px",
+      "expected": "datetime.timedelta( 34, hours=1, minutes=29 )",
+      "got": "datetime.timedelta(34)",
+      "got_raw": "datetime.timedelta(34)",
+      "exact_match": false,
+      "code_bleu": 0.3603841779658777,
+      "edit_similarity": 0.6567164179104478
+    },
+    {
+      "repo": "walles/px",
+      "expected": "get_users_at( lastline, now, datetime.datetime.now(TIMEZONE) )",
+      "got": "get_users_at(lastline, now)",
+      "got_raw": "get_users_at(lastline, now)",
+      "exact_match": false,
+      "code_bleu": 0.33635856610148585,
+      "edit_similarity": 0.6067415730337079
+    },
+    {
+      "repo": "walles/px",
+      "expected": "get_users_at( lastline, now, datetime.datetime(2016, 4, 1, 11, 8, tzinfo=TIMEZONE), )",
+      "got": "get_users_at(lastline, now)",
+      "got_raw": "get_users_at(lastline, now)",
+      "exact_match": false,
+      "code_bleu": 0.1940530189047034,
+      "edit_similarity": 0.48214285714285715
+    },
+    {
+      "repo": "walles/px",
+      "expected": "get_users_at( lastline, now, datetime.datetime(2016, 4, 3, 11, 54, tzinfo=TIMEZONE), )",
+      "got": "get_users_at(lastline, now, now)",
+      "got_raw": "get_users_at(lastline, now, now)",
+      "exact_match": false,
+      "code_bleu": 0.23096118576933522,
+      "edit_similarity": 0.5254237288135594
+    },
+    {
+      "repo": "walles/px",
+      "expected": "get_users_at( lastline, now, datetime.datetime(2016, 3, 9, 13, 26, tzinfo=TIMEZONE), )",
+      "got": "get_users_at(lastline, now)",
+      "got_raw": "get_users_at(lastline, now)",
+      "exact_match": false,
+      "code_bleu": 0.1940530189047034,
+      "edit_similarity": 0.4778761061946903
+    },
+    {
+      "repo": "walles/px",
+      "expected": "get_users_at( lastline, now, datetime.datetime(2016, 3, 26, 22, 4, tzinfo=TIMEZONE), )",
+      "got": "get_users_at(lastline, now, now)",
+      "got_raw": "get_users_at(lastline, now, now)",
+      "exact_match": false,
+      "code_bleu": 0.23096118576933522,
+      "edit_similarity": 0.5254237288135594
+    },
+    {
+      "repo": "walles/px",
+      "expected": "get_users_at( lastline, now, datetime.datetime(2016, 3, 26, 22, 9, tzinfo=TIMEZONE), )",
+      "got": "get_users_at(lastline, now, now)",
+      "got_raw": "get_users_at(lastline, now, now)",
+      "exact_match": false,
+      "code_bleu": 0.23096118576933522,
+      "edit_similarity": 0.5254237288135594
+    },
+    {
+      "repo": "walles/px",
+      "expected": "get_users_at( lastline, now, datetime.datetime(2016, 4, 4, 23, 10, tzinfo=TIMEZONE), )",
+      "got": "get_users_at(lastline, now, now)",
+      "got_raw": "get_users_at(lastline, now, now)",
+      "exact_match": false,
+      "code_bleu": 0.23096118576933522,
+      "edit_similarity": 0.5254237288135594
+    },
+    {
+      "repo": "walles/px",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "1921988 * 1024",
+      "got": "1000000000",
+      "got_raw": "1000000000",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "walles/px",
+      "expected": "371474308 * 1024",
+      "got": "1000000000",
+      "got_raw": "1000000000",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.15384615384615385
+    },
+    {
+      "repo": "walles/px",
+      "expected": "total_bytes - available_kb * 1024",
+      "got": "available_kb * 1024",
+      "got_raw": "available_kb * 1024",
+      "exact_match": false,
+      "code_bleu": 0.5045378491522287,
+      "edit_similarity": 0.7307692307692307
+    },
+    {
+      "repo": "walles/px",
+      "expected": "7",
+      "got": "7",
+      "got_raw": "7",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "4",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "p1",
+      "got": "p1",
+      "got_raw": "p1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "p2",
+      "got": "p2",
+      "got_raw": "p2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "123",
+      "got": "123",
+      "got_raw": "123",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "now",
+      "got": "now",
+      "got_raw": "now",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "1234",
+      "got": "47535",
+      "got_raw": "47535",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"0%\"",
+      "got": "\"0.00%\"",
+      "got_raw": "\"0.00%\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7272727272727273
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"6%\"",
+      "got": "5.5",
+      "got_raw": "5.5",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "pids",
+      "got": "pids",
+      "got_raw": "pids",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "0.03",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "walles/px",
+      "expected": "8",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "5",
+      "got": "5",
+      "got_raw": "5",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "close",
+      "got": "close",
+      "got_raw": "close",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "px_processinfo.to_relative_start_string(base, close)",
+      "got": "\"Mon Mar  7 09:33:11 2016\"",
+      "got_raw": "\"Mon Mar  7 09:33:11 2016\"",
+      "exact_match": false,
+      "code_bleu": 0.0625,
+      "edit_similarity": 0.1282051282051282
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[ (\"\" + str(parent_proc), parent_proc), (\" \" + str(child_proc), child_proc), ]",
+      "got": "[(\"parent\", parent_proc)]",
+      "got_raw": "[(\"parent\", parent_proc)]",
+      "exact_match": false,
+      "code_bleu": 0.12462537686556828,
+      "edit_similarity": 0.46601941747572817
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[ px_terminal.bold(\"bar(47536)\") + \": [PIPE] ->0xAda\", px_terminal.bold(\"foo(47536)\") + \": [PIPE] ->0xAda\", ]",
+      "got": "[ \"foo -> bar\", ]",
+      "got_raw": "[ \"foo -> bar\", ]",
+      "exact_match": false,
+      "code_bleu": 0.10162654963092295,
+      "edit_similarity": 0.2222222222222222
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"\"",
+      "got": "\"\"",
+      "got_raw": "\"\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"q\"",
+      "got": "\"q\"",
+      "got_raw": "\"q\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"123\"",
+      "got": "\"123\"",
+      "got_raw": "\"123\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"1234\"",
+      "got": "\"123\"",
+      "got_raw": "\"123\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"12345\"",
+      "got": "\"12345\"",
+      "got_raw": "\"12345\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"12345  \"",
+      "got": "\"12345 \"",
+      "got_raw": "\"12345 \"",
+      "exact_match": true,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.9411764705882353
+    },
+    {
+      "repo": "walles/px",
+      "expected": "mid_bold + \" \"",
+      "got": "\"12345\"",
+      "got_raw": "\"12345\"",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.19047619047619047
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[\"ab\", \"\\x1b[1m\", \"c\", \"\\x1b[22m\", \"de\"]",
+      "got": "[\"ab\", \"c\", \"de\"]",
+      "got_raw": "[\"ab\", \"c\", \"de\"]",
+      "exact_match": false,
+      "code_bleu": 0.38453868693631593,
+      "edit_similarity": 0.5964912280701754
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[ r\" PID COMMAND USERNAME CPU CPUTIME RAM COMMANDLINE\", r\"47536 \ud83d\ude00 root 0% 0.03s 0% /usr/bin/\ud83d\ude00\", ]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.04040404040404041
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[ \"\\x1b[1m PID COMMAND USERNAME CPU CPUTIME RAM COMMANDLINE\\x1b[22m\", \"47536 fluff \" + px_terminal.faint(\"root\") + \" \" + px_terminal.faint(\" 0%\") + \" \" + px_terminal.bold(\" 0.03s\") + \" \" + px_terminal.faint(\" 0%\") + \" /usr/bin/fluff 1234\", ]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.028314455119112403,
+      "edit_similarity": 0.01646090534979424
+    },
+    {
+      "repo": "walles/px",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"--\"",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "SCREEN_ROWS",
+      "got": "11",
+      "got_raw": "11",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "px_top.CMD_QUIT",
+      "got": "\"q\"",
+      "got_raw": "\"q\"",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "px_top.CMD_RESIZE",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "lines[0].replace(CSI, \"CSI\")",
+      "got": "lines[0]",
+      "got_raw": "lines[0]",
+      "exact_match": false,
+      "code_bleu": 0.36363636363636365,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "walles/px",
+      "expected": "lines[-1].replace(CSI, \"CSI\")",
+      "got": "lines[-1]",
+      "got_raw": "lines[-1]",
+      "exact_match": false,
+      "code_bleu": 0.4166666666666667,
+      "edit_similarity": 0.47368421052631576
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[\"parent(1)\", \"  child(2)\"]",
+      "got": "[px_process.PxProcess(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1",
+      "got_raw": "[px_process.PxProcess(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1",
+      "exact_match": false,
+      "code_bleu": 0.015393994773377797,
+      "edit_similarity": 0.11764705882352941
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[f\"Lumpur... ({px_terminal.bold('3\u00d7')})\"]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.29730177875068026,
+      "edit_similarity": 0.09302325581395349
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[\"root(1)\", \"  find-me(2)\", \"  uninteresting(3)\"]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.16988673071467442,
+      "edit_similarity": 0.0784313725490196
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[\"root(1)\", f\"  {px_terminal.bold('find-me')}(2)\"]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.07692307692307693
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[\"root(1)\", f\"  process... ({px_terminal.bold('2\u00d7')})\"]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.07017543859649122
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[ \"Kuala(1)\" ]",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.13333333333333333
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[ \"root(1)\", f\" {px_terminal.bold('process')}(2)\", f\" {px_terminal.bold('process')}(3)\", ]",
+      "got": "[px_process.PxProcess(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1",
+      "got_raw": "[px_process.PxProcess(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1",
+      "exact_match": false,
+      "code_bleu": 0.017490954988811324,
+      "edit_similarity": 0.18025751072961374
+    },
+    {
+      "repo": "walles/px",
+      "expected": "[ \"root(1)\", f\" process... ({px_terminal.bold('4\u00d7')})\", \" process(4)\", \" subprocess(7)\", ]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.1081097377275201,
+      "edit_similarity": 0.043478260869565216
+    },
+    {
+      "repo": "walles/px",
+      "expected": "(\"0KB\", \"7KB\")",
+      "got": "(\"0B\", \"7.0KB\")",
+      "got_raw": "(\"0B\", \"7.0KB\")",
+      "exact_match": false,
+      "code_bleu": 0.32466791547509893,
+      "edit_similarity": 0.896551724137931
+    },
+    {
+      "repo": "walles/px",
+      "expected": "(\"0MB\", \"7MB\")",
+      "got": "(\"0MB\", \"7MB\")",
+      "got_raw": "(\"0MB\", \"7MB\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "(\"0GB\", \"7GB\")",
+      "got": "(\"0GB\", \"7GB\")",
+      "got_raw": "(\"0GB\", \"7GB\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "(\"0TB\", \"7TB\")",
+      "got": "(\"0TB\", \"7TB\")",
+      "got_raw": "(\"0TB\", \"7TB\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "(\"0B\", \"7167B\")",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.18072040072196896,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "1235",
+      "got": "\"1235\"",
+      "got_raw": "\"1235\"",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "walles/px",
+      "expected": "args)",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "walles/px",
+      "expected": "\"kalas\"",
+      "got": "\"kalas\"",
+      "got_raw": "\"kalas\"",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "px.__doc__)",
+      "got": "px.__doc__)",
+      "got_raw": "px.__doc__)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "walles/px",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "url)",
+      "got": "url)",
+      "got_raw": "url)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "res)",
+      "got": "res)",
+      "got_raw": "res)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "group)",
+      "got": "group.id)",
+      "got_raw": "group.id)",
+      "exact_match": false,
+      "code_bleu": 0.3976353643835253,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "users)",
+      "got": "users[0])",
+      "got_raw": "users[0])",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "result)",
+      "got": "result)",
+      "got_raw": "result)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "ticket)",
+      "got": "\"123456\"",
+      "got_raw": "\"123456\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "client2)",
+      "got": "client2)",
+      "got_raw": "client2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "card_ext)",
+      "got": "card_ext)",
+      "got_raw": "card_ext)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "len(res))",
+      "got": "len(res))",
+      "got_raw": "len(res))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "group_id)",
+      "got": "group_id)",
+      "got_raw": "group_id)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "authorize_url)",
+      "got": "redirect_uri)",
+      "got_raw": "redirect_uri)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.4444444444444444
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "res[\"openid\"])",
+      "got": "res[\"openid\"])",
+      "got_raw": "res[\"openid\"])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "result[\"errcode\"])",
+      "got": "result)",
+      "got_raw": "result)",
+      "exact_match": false,
+      "code_bleu": 0.23784142300054423,
+      "edit_similarity": 0.56
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "token[\"expires_in\"])",
+      "got": "token[\"expires_in\"])",
+      "got_raw": "token[\"expires_in\"])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "res[\"access_token\"])",
+      "got": "res)",
+      "got_raw": "res)",
+      "exact_match": false,
+      "code_bleu": 0.23784142300054423,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "result[\"expires_in\"])",
+      "got": "result[\"expires_in\"])",
+      "got_raw": "result[\"expires_in\"])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "result[\"option_name\"])",
+      "got": "result)",
+      "got_raw": "result)",
+      "exact_match": false,
+      "code_bleu": 0.23784142300054423,
+      "edit_similarity": 0.4827586206896552
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "result[\"option_value\"])",
+      "got": "result[\"option_value\"])",
+      "got_raw": "result[\"option_value\"])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "e.response.status_code)",
+      "got": "e.errcode)",
+      "got_raw": "e.errcode)",
+      "exact_match": false,
+      "code_bleu": 0.3387551654364098,
+      "edit_similarity": 0.48484848484848486
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "result[\"pre_auth_code\"])",
+      "got": "result)",
+      "got_raw": "result)",
+      "exact_match": false,
+      "code_bleu": 0.23784142300054423,
+      "edit_similarity": 0.45161290322580644
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "self.client.access_token)",
+      "got": "self.client.app_id)",
+      "got_raw": "self.client.app_id)",
+      "exact_match": false,
+      "code_bleu": 0.6434588841607617,
+      "edit_similarity": 0.6818181818181818
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "token[\"component_access_token\"])",
+      "got": "token)",
+      "got_raw": "token)",
+      "exact_match": false,
+      "code_bleu": 0.23784142300054423,
+      "edit_similarity": 0.3157894736842105
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "result[\"authorizer_access_token\"])",
+      "got": "result[\"authorizer_access_token\"])",
+      "got_raw": "result[\"authorizer_access_token\"])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "result[\"authorizer_refresh_token\"])",
+      "got": "result[\"authorizer_refresh_token\"])",
+      "got_raw": "result[\"authorizer_refresh_token\"])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "result[\"authorizer_info\"][\"alias\"])",
+      "got": "result[\"authorizer_info\"][\"authorizer_name\"])",
+      "got_raw": "result[\"authorizer_info\"][\"authorizer_name\"])",
+      "exact_match": false,
+      "code_bleu": 0.6606328636027614,
+      "edit_similarity": 0.825
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "inspect.ismethod(self.client.fetch_access_token))",
+      "got": "True)",
+      "got_raw": "True)",
+      "exact_match": false,
+      "code_bleu": 0.097688175620322,
+      "edit_similarity": 0.07407407407407407
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "result[\"authorization_info\"][\"authorizer_appid\"])",
+      "got": "result[\"authorizer_appid\"])",
+      "got_raw": "result[\"authorizer_appid\"])",
+      "exact_match": false,
+      "code_bleu": 0.5255602595335715,
+      "edit_similarity": 0.7105263157894737
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "WeChatErrorCode(-1))",
+      "got": "10001)",
+      "got_raw": "10001)",
+      "exact_match": false,
+      "code_bleu": 0.17909498863725698,
+      "edit_similarity": 0.15384615384615385
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "WeChatErrorCode.SYSTEM_ERROR.value)",
+      "got": "WeChatErrorCode.INVALID_ACCESS_TOKEN)",
+      "got_raw": "WeChatErrorCode.INVALID_ACCESS_TOKEN)",
+      "exact_match": false,
+      "code_bleu": 0.3387551654364098,
+      "edit_similarity": 0.5555555555555556
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "WeChatErrorCode.UNAUTHORIZED_API.value)",
+      "got": "WeChatErrorCode.INVALID_APPID.value)",
+      "got_raw": "WeChatErrorCode.INVALID_APPID.value)",
+      "exact_match": false,
+      "code_bleu": 0.488923022434901,
+      "edit_similarity": 0.7733333333333333
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "WeChatErrorCode.EXPIRED_ACCESS_TOKEN.value)",
+      "got": "WeChatErrorCode.INVALID_ACCESS_TOKEN.value)",
+      "got_raw": "WeChatErrorCode.INVALID_ACCESS_TOKEN.value)",
+      "exact_match": false,
+      "code_bleu": 0.488923022434901,
+      "edit_similarity": 0.8837209302325582
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply)",
+      "got": "reply)",
+      "got_raw": "reply)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply.type)",
+      "got": "reply.type)",
+      "got_raw": "reply.type)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply.target)",
+      "got": "reply.from_user)",
+      "got_raw": "reply.from_user)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.6206896551724138
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply.source)",
+      "got": "reply.from_user)",
+      "got_raw": "reply.from_user)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.6206896551724138
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply.content)",
+      "got": "reply.content)",
+      "got_raw": "reply.content)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "AttributeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6153846153846154
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "isinstance(reply, str))",
+      "got": "reply.text)",
+      "got_raw": "reply.text)",
+      "exact_match": false,
+      "code_bleu": 0.2272202082191573,
+      "edit_similarity": 0.4117647058823529
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "isinstance(reply, EmptyReply))",
+      "got": "isinstance(reply, EmptyReply))",
+      "got_raw": "isinstance(reply, EmptyReply))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "encrypted)",
+      "got": "encrypted)",
+      "got_raw": "encrypted)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "data[\"refund_id\"])",
+      "got": "data[\"refund_id\"])",
+      "got_raw": "data[\"refund_id\"])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg_dict[\"Content\"])",
+      "got": "msg_dict[\"ToUserName\"])",
+      "got_raw": "msg_dict[\"ToUserName\"])",
+      "exact_match": false,
+      "code_bleu": 0.45180100180492244,
+      "edit_similarity": 0.6976744186046512
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "data[\"out_trade_no\"])",
+      "got": "data[\"trade_no\"])",
+      "got_raw": "data[\"trade_no\"])",
+      "exact_match": false,
+      "code_bleu": 0.45180100180492244,
+      "edit_similarity": 0.8947368421052632
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "data[\"out_refund_no\"])",
+      "got": "data[\"appid\"])",
+      "got_raw": "data[\"appid\"])",
+      "exact_match": false,
+      "code_bleu": 0.45180100180492244,
+      "edit_similarity": 0.5555555555555556
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "data[\"transaction_id\"])",
+      "got": "data[\"result_code\"])",
+      "got_raw": "data[\"result_code\"])",
+      "exact_match": false,
+      "code_bleu": 0.45180100180492244,
+      "edit_similarity": 0.6511627906976745
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg_dict[\"FromUserName\"])",
+      "got": "msg_dict[\"MsgType\"])",
+      "got_raw": "msg_dict[\"MsgType\"])",
+      "exact_match": false,
+      "code_bleu": 0.45180100180492244,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "InvalidSignatureException)",
+      "got": "with",
+      "got_raw": "with self.assertRaises(InvalidSignatureException)",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.13333333333333333
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "decrypted[\"watermark\"][\"appid\"])",
+      "got": "decrypted[\"FromUserName\"])",
+      "got_raw": "decrypted[\"FromUserName\"])",
+      "exact_match": false,
+      "code_bleu": 0.28237562612807654,
+      "edit_similarity": 0.5862068965517241
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "xml2)",
+      "got": "xml2)",
+      "got_raw": "xml2)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "response)",
+      "got": "response)",
+      "got_raw": "response)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "event.id)",
+      "got": "event.msg_id)",
+      "got_raw": "event.msg_id)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.8181818181818182
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "event.key)",
+      "got": "event.event_key)",
+      "got_raw": "event.event_key)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.7692307692307693
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "event.url)",
+      "got": "event.event_key)",
+      "got_raw": "event.event_key)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.5384615384615384
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "\"by time\")",
+      "got": "\"by time\")",
+      "got_raw": "\"by time\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "event.count)",
+      "got": "event.count)",
+      "got_raw": "event.count)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "event.scale)",
+      "got": "event.scale)",
+      "got_raw": "event.scale)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "event.label)",
+      "got": "",
+      "got_raw": "",
+      "exact_match": false,
+      "code_bleu": 0.0,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "event.bssid)",
+      "got": "event.device_no)",
+      "got_raw": "event.device_no)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.5714285714285714
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "event.source)",
+      "got": "event.from_user)",
+      "got_raw": "event.from_user)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.6206896551724138
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "event.target)",
+      "got": "event.target)",
+      "got_raw": "event.target)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "event.ticket)",
+      "got": "event.ticket)",
+      "got_raw": "event.ticket)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "res[\"errcode\"])",
+      "got": "res[\"errcode\"])",
+      "got_raw": "res[\"errcode\"])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "len(external_contact_list))",
+      "got": "len(external_contact_list))",
+      "got_raw": "len(external_contact_list))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "xml)",
+      "got": "xml)",
+      "got_raw": "xml)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "field.to_xml(value))",
+      "got": "field.to_xml(value))",
+      "got_raw": "field.to_xml(value))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "field.to_xml(content))",
+      "got": "field.to_xml(content))",
+      "got_raw": "field.to_xml(content))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "field.to_xml(articles))",
+      "got": "field.to_xml(articles))",
+      "got_raw": "field.to_xml(articles))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "client)",
+      "got": "client.access_token)",
+      "got_raw": "client.access_token)",
+      "exact_match": false,
+      "code_bleu": 0.3976353643835253,
+      "edit_similarity": 0.5185185185185185
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "res[\"errcode\"])",
+      "got": "res[\"errcode\"])",
+      "got_raw": "res[\"errcode\"])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "token[\"expires_in\"])",
+      "got": "token[\"expires_in\"])",
+      "got_raw": "token[\"expires_in\"])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "token[\"access_token\"])",
+      "got": "token)",
+      "got_raw": "token)",
+      "exact_match": false,
+      "code_bleu": 0.23784142300054423,
+      "edit_similarity": 0.42857142857142855
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "self.client.access_token)",
+      "got": "self.app_id)",
+      "got_raw": "self.app_id)",
+      "exact_match": false,
+      "code_bleu": 0.3387551654364098,
+      "edit_similarity": 0.43243243243243246
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.id)",
+      "got": "msg.msg_id)",
+      "got_raw": "msg.msg_id)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.7777777777777778
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.url)",
+      "got": "msg.url)",
+      "got_raw": "msg.url)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.image)",
+      "got": "msg.pic_url)",
+      "got_raw": "msg.pic_url)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.5454545454545454
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.scale)",
+      "got": "msg.scale)",
+      "got_raw": "msg.scale)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.label)",
+      "got": "msg.label)",
+      "got_raw": "msg.label)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.title)",
+      "got": "msg.title)",
+      "got_raw": "msg.title)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.source)",
+      "got": "msg.from_user)",
+      "got_raw": "msg.from_user)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.56
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.target)",
+      "got": "msg.target)",
+      "got_raw": "msg.target)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.format)",
+      "got": "msg.format)",
+      "got_raw": "msg.format)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.app_id)",
+      "got": "msg.appid)",
+      "got_raw": "msg.appid)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.9523809523809523
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.content)",
+      "got": "msg.content)",
+      "got_raw": "msg.content)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.media_id)",
+      "got": "msg.media_id)",
+      "got_raw": "msg.media_id)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.page_path)",
+      "got": "msg.page_path)",
+      "got_raw": "msg.page_path)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.thumb_url)",
+      "got": "msg.thumb_url)",
+      "got_raw": "msg.thumb_url)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.location_x)",
+      "got": "msg.location_x)",
+      "got_raw": "msg.location_x)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.location_y)",
+      "got": "msg.location_y)",
+      "got_raw": "msg.location_y)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.recognition)",
+      "got": "msg.recognition)",
+      "got_raw": "msg.recognition)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.description)",
+      "got": "msg.description)",
+      "got_raw": "msg.description)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "url)",
+      "got": "\"wechatpy\"",
+      "got_raw": "\"wechatpy\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "res)",
+      "got": "res)",
+      "got_raw": "res)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "authorize_url)",
+      "got": "\"wechatpy\"",
+      "got_raw": "\"wechatpy\"",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.08333333333333333
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "res[\"openid\"])",
+      "got": "res[\"openid\"])",
+      "got_raw": "res[\"openid\"])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "res[\"access_token\"])",
+      "got": "res.access_token)",
+      "got_raw": "res.access_token)",
+      "exact_match": false,
+      "code_bleu": 0.3181082915068203,
+      "edit_similarity": 0.8648648648648649
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "e.response.status_code)",
+      "got": "e.errcode)",
+      "got_raw": "e.errcode)",
+      "exact_match": false,
+      "code_bleu": 0.3387551654364098,
+      "edit_similarity": 0.48484848484848486
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.key)",
+      "got": "msg.msgid)",
+      "got_raw": "msg.msgid)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.5555555555555556
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.url)",
+      "got": "msg.event_key)",
+      "got_raw": "msg.event_key)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.45454545454545453
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.type)",
+      "got": "msg.type)",
+      "got_raw": "msg.type)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.code)",
+      "got": "msg.code)",
+      "got_raw": "msg.code)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "xml)",
+      "got": "xml)",
+      "got_raw": "xml)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "sign)",
+      "got": "sign)",
+      "got_raw": "sign)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "sign)",
+      "got": "sign)",
+      "got_raw": "sign)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "response)",
+      "got": "response)",
+      "got_raw": "response)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "[{\"encrypt_certificate\": \"\", \"serial_no\": \"\"}, {\"encrypt_certificate\": \"\", \"serial_no\": \"\"}])",
+      "got": ".client.update_certificates()",
+      "got_raw": ".client.update_certificates()",
+      "exact_match": false,
+      "code_bleu": 0.06024433945679423,
+      "edit_similarity": 0.2786885245901639
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "response)",
+      "got": "response)",
+      "got_raw": "response)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "\"SUCCESS\")",
+      "got": "\"SUCCESS\")",
+      "got_raw": "\"SUCCESS\")",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "response[\"data\"])",
+      "got": "response[\"data\"])",
+      "got_raw": "response[\"data\"])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply.image)",
+      "got": "reply.image)",
+      "got_raw": "reply.image)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply.voice)",
+      "got": "reply.voice)",
+      "got_raw": "reply.voice)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply.title)",
+      "got": "reply.title)",
+      "got_raw": "reply.title)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply.source)",
+      "got": "reply.source)",
+      "got_raw": "reply.source)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply.target)",
+      "got": "reply.target)",
+      "got_raw": "reply.target)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply.media_id)",
+      "got": "reply.image_url)",
+      "got_raw": "reply.image_url)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.6451612903225806
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply.render())",
+      "got": "reply.content)",
+      "got_raw": "reply.content)",
+      "exact_match": false,
+      "code_bleu": 0.3387551654364098,
+      "edit_similarity": 0.6206896551724138
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply.music_url)",
+      "got": "reply.music_url)",
+      "got_raw": "reply.music_url)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "create_time in r)",
+      "got": "create_time in r)",
+      "got_raw": "create_time in r)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "url)",
+      "got": "url)",
+      "got_raw": "url)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "token[\"expires_in\"])",
+      "got": "token[\"expires_in\"])",
+      "got_raw": "token[\"expires_in\"])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "client.access_token)",
+      "got": "client.access_token)",
+      "got_raw": "client.access_token)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "token[\"access_token\"])",
+      "got": "token)",
+      "got_raw": "token)",
+      "exact_match": false,
+      "code_bleu": 0.23784142300054423,
+      "edit_similarity": 0.42857142857142855
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "isinstance(client.session, RedisStorage))",
+      "got": "client.session)",
+      "got_raw": "client.session)",
+      "exact_match": false,
+      "code_bleu": 0.31426968052735443,
+      "edit_similarity": 0.5357142857142857
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "isinstance(client.session, MemoryStorage))",
+      "got": "isinstance(client.session_storage, MemoryStorage))",
+      "got_raw": "isinstance(client.session_storage, MemoryStorage))",
+      "exact_match": false,
+      "code_bleu": 0.6580370064762462,
+      "edit_similarity": 0.9130434782608695
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "isinstance(client.session, MemcachedStorage))",
+      "got": "client.session)",
+      "got_raw": "client.session)",
+      "exact_match": false,
+      "code_bleu": 0.31426968052735443,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "obj.xxx)",
+      "got": "obj.xxx)",
+      "got_raw": "obj.xxx)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "signature)",
+      "got": "signature)",
+      "got_raw": "signature)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "obj.xxx is None)",
+      "got": "obj)",
+      "got_raw": "obj)",
+      "exact_match": false,
+      "code_bleu": 0.19820118583378682,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "InvalidSignatureException)",
+      "got": "with",
+      "got_raw": "with pytest.raises(InvalidSignatureException)",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.13333333333333333
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "target_string.encode(\"utf-8\"))",
+      "got": "target_string",
+      "got_raw": "target_string",
+      "exact_match": false,
+      "code_bleu": 0.08494336535733721,
+      "edit_similarity": 0.6046511627906976
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "res)",
+      "got": "res)",
+      "got_raw": "res)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "100)",
+      "got": "[])",
+      "got_raw": "[])",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.2857142857142857
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "dict)",
+      "got": "list)",
+      "got_raw": "list)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "client)",
+      "got": "client.access_token)",
+      "got_raw": "client.access_token)",
+      "exact_match": false,
+      "code_bleu": 0.3976353643835253,
+      "edit_similarity": 0.5185185185185185
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "user_id)",
+      "got": "user_id)",
+      "got_raw": "user_id)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "len(res))",
+      "got": "len(res))",
+      "got_raw": "len(res))",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "signature)",
+      "got": "signature)",
+      "got_raw": "signature)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "res[\"id\"])",
+      "got": "res[\"id\"])",
+      "got_raw": "res[\"id\"])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply)",
+      "got": "reply)",
+      "got_raw": "reply)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply.type)",
+      "got": "reply.type)",
+      "got_raw": "reply.type)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply.agent)",
+      "got": "len(reply.data))",
+      "got_raw": "len(reply.data))",
+      "exact_match": false,
+      "code_bleu": 0.2626909894424158,
+      "edit_similarity": 0.6428571428571429
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply.target)",
+      "got": "reply.from_user)",
+      "got_raw": "reply.from_user)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.6206896551724138
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply.source)",
+      "got": "reply.from_user)",
+      "got_raw": "reply.from_user)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.6206896551724138
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply.content)",
+      "got": "reply.content)",
+      "got_raw": "reply.content)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "reply is None)",
+      "got": "reply is None)",
+      "got_raw": "reply is None)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "AttributeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6153846153846154
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "isinstance(reply, str))",
+      "got": "reply)",
+      "got_raw": "reply)",
+      "exact_match": false,
+      "code_bleu": 0.16988673071467442,
+      "edit_similarity": 0.41379310344827586
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.key)",
+      "got": "msg.key)",
+      "got_raw": "msg.key)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.url)",
+      "got": "msg.event_key)",
+      "got_raw": "msg.event_key)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.45454545454545453
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.type)",
+      "got": "msg.msg_type)",
+      "got_raw": "msg.msg_type)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.8181818181818182
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.agent)",
+      "got": "msg.agent_id)",
+      "got_raw": "msg.agent_id)",
+      "exact_match": false,
+      "code_bleu": 0.5081327481546147,
+      "edit_similarity": 0.8695652173913043
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.event)",
+      "got": "msg.event)",
+      "got_raw": "msg.event)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wechatpy/wechatpy",
+      "expected": "msg.job_id)",
+      "got": "msg.job_id)",
+      "got_raw": "msg.job_id)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "ref2.ref",
+      "got": "ref2.ref",
+      "got_raw": "ref2.ref",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "HTTPStatus.BAD_REQUEST",
+      "got_raw": "HTTPStatus.BAD_REQUEST",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.6285714285714286
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.OK",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "refresh['jti']",
+      "got": "refresh['jti']",
+      "got_raw": "refresh['jti']",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "ContentType.xml",
+      "got": "ContentType.xml",
+      "got_raw": "ContentType.xml",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "request.headers",
+      "got": "request.headers",
+      "got_raw": "request.headers",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "dict[str, int])",
+      "got": "Path)",
+      "got_raw": "Path)",
+      "exact_match": false,
+      "code_bleu": 0.153509990260506,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "dict[str, int])",
+      "got": "Body)",
+      "got_raw": "Body)",
+      "exact_match": false,
+      "code_bleu": 0.153509990260506,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "dict[str, str])",
+      "got": "dict[str, str])",
+      "got_raw": "dict[str, str])",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "ContentType.json",
+      "got": "str(ContentType.json)",
+      "got_raw": "str(ContentType.json)",
+      "exact_match": false,
+      "code_bleu": 0.4111336169005197,
+      "edit_similarity": 0.8648648648648649
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.FOUND",
+      "got": "HTTPStatus.FOUND",
+      "got_raw": "HTTPStatus.FOUND",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "HTTPStatus.OK",
+      "got_raw": "HTTPStatus.OK",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.7096774193548387
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "HTTPStatus.OK",
+      "got_raw": "HTTPStatus.OK",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.7096774193548387
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "HTTPStatus.OK",
+      "got_raw": "HTTPStatus.OK",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.7096774193548387
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "self.request.FILES",
+      "got": "self.parsed_file_metadata.model_fields",
+      "got_raw": "self.parsed_file_metadata.model_fields",
+      "exact_match": false,
+      "code_bleu": 0.3860973950960897,
+      "edit_similarity": 0.32142857142857145
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "422",
+      "got_raw": "422",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "422",
+      "got_raw": "422",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "OpenAPIFormat.IPV4",
+      "got": "'ipv4'",
+      "got_raw": "'ipv4'",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CREATED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CONFLICT",
+      "got": "422",
+      "got_raw": "422",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.ACCEPTED",
+      "got": "200",
+      "got_raw": "200",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CONFLICT",
+      "got": "404",
+      "got_raw": "404",
+      "exact_match": false,
+      "code_bleu": 0.16666666666666666,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "OpenAPIType.INTEGER",
+      "got": "OpenAPIType.INTEGER",
+      "got_raw": "OpenAPIType.INTEGER",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.CONFLICT",
+      "got": "HTTPStatus.CONFLICT",
+      "got_raw": "HTTPStatus.CONFLICT",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.FORBIDDEN",
+      "got": "HTTPStatus.FORBIDDEN",
+      "got_raw": "HTTPStatus.FORBIDDEN",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "OpenAPIFormat.BINARY",
+      "got": "OpenAPIFormat.BINARY",
+      "got_raw": "OpenAPIFormat.BINARY",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "HTTPStatus.FORBIDDEN",
+      "got": "HTTPStatus.NOT_ACCEPTABLE",
+      "got_raw": "HTTPStatus.NOT_ACCEPTABLE",
+      "exact_match": false,
+      "code_bleu": 0.5946035575013605,
+      "edit_similarity": 0.5333333333333333
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "[]",
+      "got": "{'foo':",
+      "got_raw": "{'foo': 'bar'}",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "[]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "()",
+      "got": "expected_capture",
+      "got_raw": "expected_capture",
+      "exact_match": false,
+      "code_bleu": 0.25,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "['a']",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.30120066786994826,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "[1, 2]",
+      "got": "{ \"cookies\": { \"new\": { \"new_cookie\": \"new_cookie_value\" }, }, }, }",
+      "got_raw": "{ \"cookies\": { \"new\": { \"new_cookie\": \"new_cookie_value\" }, }, }, }",
+      "exact_match": false,
+      "code_bleu": 0.07223943354597204,
+      "edit_similarity": 0.0547945205479452
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "[1, 2]",
+      "got": "{'status': 'ok'}",
+      "got_raw": "{'status': 'ok'}",
+      "exact_match": false,
+      "code_bleu": 0.2295748846661433,
+      "edit_similarity": 0.09090909090909091
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "[1, 2]",
+      "got": "{'foo': 'bar'}",
+      "got_raw": "{'foo': 'bar'}",
+      "exact_match": false,
+      "code_bleu": 0.2295748846661433,
+      "edit_similarity": 0.1
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "['a', 'b']",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.18072040072196896,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'detail'}",
+      "got": "{'error'}",
+      "got_raw": "{'error'}",
+      "exact_match": false,
+      "code_bleu": 0.49999999999999994,
+      "edit_similarity": 0.5263157894736842
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "('', (), {})",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": false,
+      "code_bleu": 0.05555555555555555,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "(_BodyModel,)",
+      "got": "Body",
+      "got_raw": "Body",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.47058823529411764
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "['card', 'big']",
+      "got": "json.loads(response.content)",
+      "got_raw": "json.loads(response.content)",
+      "exact_match": false,
+      "code_bleu": 0.13485111859503685,
+      "edit_similarity": 0.09302325581395349
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{HTTPStatus.OK}",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.18072040072196896,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'user': 'name'}",
+      "got": "{'foo': 'bar'}",
+      "got_raw": "{'foo': 'bar'}",
+      "exact_match": false,
+      "code_bleu": 0.32466791547509893,
+      "edit_similarity": 0.6
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'key': 'value'}",
+      "got": "'{\"key\": \"value\"}'",
+      "got_raw": "'{\"key\": \"value\"}'",
+      "exact_match": false,
+      "code_bleu": 0.1,
+      "edit_similarity": 0.7058823529411765
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'result': 'done'}",
+      "got": "{'status': 'ok'}",
+      "got_raw": "{'status': 'ok'}",
+      "exact_match": false,
+      "code_bleu": 0.32466791547509893,
+      "edit_similarity": 0.6470588235294118
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "[{'noScheme': []}]",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.1767766952966369,
+      "edit_similarity": 0.2
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "['first', 'second']",
+      "got": "['first', 'second']",
+      "got_raw": "['first', 'second']",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "[{'testScheme': []}]",
+      "got": "[SecurityRequirement(schemes=['Bearer'])]",
+      "got_raw": "[SecurityRequirement(schemes=['Bearer'])]",
+      "exact_match": false,
+      "code_bleu": 0.15851165692617156,
+      "edit_similarity": 0.32786885245901637
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'username': 'admin'}",
+      "got": "{'foo': 'bar'}",
+      "got_raw": "{'foo': 'bar'}",
+      "exact_match": false,
+      "code_bleu": 0.32466791547509893,
+      "edit_similarity": 0.5142857142857142
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'message': 'Success'}",
+      "got": "{'message': 'Custom header middleware works'}",
+      "got_raw": "{'message': 'Custom header middleware works'}",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.5373134328358209
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'X-API-Token': '123'}",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.23784142300054423,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'full_name': 'Example'}",
+      "got": "{'forward_ref': 'ForwardRef'}",
+      "got_raw": "{'forward_ref': 'ForwardRef'}",
+      "exact_match": false,
+      "code_bleu": 0.32466791547509893,
+      "edit_similarity": 0.49056603773584906
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'session_id': 'unique'}",
+      "got": "{'whatever': 'yes'}",
+      "got_raw": "{'whatever': 'yes'}",
+      "exact_match": false,
+      "code_bleu": 0.32466791547509893,
+      "edit_similarity": 0.46511627906976744
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'user_id': str(user.pk)}",
+      "got": "{'username': user.username}",
+      "got_raw": "{'username': user.username}",
+      "exact_match": false,
+      "code_bleu": 0.20350075793887676,
+      "edit_similarity": 0.5769230769230769
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "[{'name': _EXAMPLE_NAME}]",
+      "got": "[_EXAMPLE_NAME]",
+      "got_raw": "[_EXAMPLE_NAME]",
+      "exact_match": false,
+      "code_bleu": 0.23026498539075899,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'query': expected_query_value}",
+      "got": "{'query': expected_query_value}",
+      "got_raw": "{'query': expected_query_value}",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "{'username': 'validated-admin'}",
+      "got": "{'foo': 'bar'}",
+      "got_raw": "{'foo': 'bar'}",
+      "exact_match": false,
+      "code_bleu": 0.32466791547509893,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "None",
+      "got": "None",
+      "got_raw": "None",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "str",
+      "got": "str",
+      "got_raw": "str",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "int",
+      "got": "int",
+      "got_raw": "int",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "iss",
+      "got": "iss",
+      "got_raw": "iss",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "bool",
+      "got": "Any",
+      "got_raw": "Any",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "props",
+      "got": "props",
+      "got_raw": "props",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "header",
+      "got": "header",
+      "got_raw": "header",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "regular",
+      "got": "{",
+      "got_raw": "{ 'regular': [], }",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "headers",
+      "got": "headers",
+      "got_raw": "headers",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "expected",
+      "got": "{",
+      "got_raw": "{ 'error': 'Payment required', }",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "username",
+      "got": "{'username':",
+      "got_raw": "{'username': username}",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.8
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "expected",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "_MAXIMUM",
+      "got": "10",
+      "got_raw": "10",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "expected",
+      "got": "{'error':",
+      "got_raw": "{'error': expected}",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.11764705882352941
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "audience",
+      "got": "audience",
+      "got_raw": "audience",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "expected",
+      "got": "True(expected)",
+      "got_raw": "True(expected)",
+      "exact_match": false,
+      "code_bleu": 0.35930411196308426,
+      "edit_similarity": 0.7272727272727273
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "last_name",
+      "got": "{",
+      "got_raw": "{ 'query': last_name, }",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "field_map",
+      "got": "field_map",
+      "got_raw": "field_map",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "request_id",
+      "got": "request_id",
+      "got_raw": "request_id",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "TypeError)",
+      "got": "ValueError)",
+      "got_raw": "ValueError)",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "controller",
+      "got": "controller",
+      "got_raw": "controller",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "status_code",
+      "got": "status_code",
+      "got_raw": "status_code",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "status_code",
+      "got": "status_code",
+      "got_raw": "status_code",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "status_code",
+      "got": "status_code",
+      "got_raw": "status_code",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "_MIN_LENGTH",
+      "got": "_MIN_LENGTH",
+      "got_raw": "_MIN_LENGTH",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "b''",
+      "got": "b''",
+      "got_raw": "b''",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "b''",
+      "got": "b'{\"foo\":",
+      "got_raw": "b'{\"foo\": \"bar\"}'",
+      "exact_match": false,
+      "code_bleu": 0.2730120862709067,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "b''",
+      "got": "b'{}'",
+      "got_raw": "b'{}'",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "b''",
+      "got": "b''",
+      "got_raw": "b''",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "b''",
+      "got": "b'{}'",
+      "got_raw": "b'{}'",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "b''",
+      "got": "b\"\"",
+      "got_raw": "b\"\"",
+      "exact_match": false,
+      "code_bleu": 0.537284965911771,
+      "edit_similarity": 0.3333333333333333
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "b''",
+      "got": "b''",
+      "got_raw": "b''",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'field'",
+      "got": "'extra'",
+      "got_raw": "'extra'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.42857142857142855
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'HS256'",
+      "got": "'HS256'",
+      "got_raw": "'HS256'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'authed'",
+      "got": "{'authed':",
+      "got_raw": "{'authed': True}",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.8888888888888888
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'authed'",
+      "got": "{'success':",
+      "got_raw": "{'success': True}",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.42105263157894735
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'text/html'",
+      "got": "'text/html;",
+      "got_raw": "'text/html; charset=utf-8'",
+      "exact_match": false,
+      "code_bleu": 0.2295748846661433,
+      "edit_similarity": 0.9090909090909091
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'not authed'",
+      "got": "{'message': 'not authed'}",
+      "got_raw": "{'message': 'not authed'}",
+      "exact_match": false,
+      "code_bleu": 0.2730120862709067,
+      "edit_similarity": 0.6486486486486487
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'CustomValue'",
+      "got": "'CustomValue'",
+      "got_raw": "'CustomValue'",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'value_error'",
+      "got": "'error'",
+      "got_raw": "'error'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.7
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'value_error'",
+      "got": "'ValidationError'",
+      "got_raw": "'ValidationError'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.5333333333333333
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'none handled'",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'Error message'",
+      "got": "{ 'error': 'Invalid data'",
+      "got_raw": "{ 'error': 'Invalid data', }",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.3
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'customGetUser'",
+      "got": "'GET'",
+      "got_raw": "'GET'",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.3
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "'{\"key\":\"value\"}'",
+      "got": "request_data['extra']",
+      "got_raw": "request_data['extra']",
+      "exact_match": false,
+      "code_bleu": 0.3021375397356768,
+      "edit_similarity": 0.21052631578947367
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot(1)",
+      "got": "{",
+      "got_raw": "{ 'error': 'Conflict', }",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "list(expected)",
+      "got": "expected",
+      "got_raw": "expected",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.7272727272727273
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot(['a'])",
+      "got": "{",
+      "got_raw": "{ 'foo': 'bar', }",
+      "exact_match": false,
+      "code_bleu": 0.08333333333333333,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "len(operation_ids)",
+      "got": "4",
+      "got_raw": "4",
+      "exact_match": false,
+      "code_bleu": 0.125,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "request_data.keys()",
+      "got": "request_data.keys()",
+      "got_raw": "request_data.keys()",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "str(exc_info.value)",
+      "got": "str(exc_info.value)",
+      "got_raw": "str(exc_info.value)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "sorted(request_data)",
+      "got": "request_data",
+      "got_raw": "request_data",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.75
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot({'jwt': []})",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.15713484026367722,
+      "edit_similarity": 0.17391304347826086
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot({'field': 1})",
+      "got": "{'field': '1'}",
+      "got_raw": "{'field': '1'}",
+      "exact_match": false,
+      "code_bleu": 0.3358031036948569,
+      "edit_similarity": 0.6666666666666666
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot(b'{\"a\": \"b\"}')",
+      "got": "json.dumps({'a': 'b'})",
+      "got_raw": "json.dumps({'a': 'b'})",
+      "exact_match": false,
+      "code_bleu": 0.13950796967929133,
+      "edit_similarity": 0.4888888888888889
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "len(request_data['user'])",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": false,
+      "code_bleu": 0.07142857142857142,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot({'http_basic': []})",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.15713484026367722,
+      "edit_similarity": 0.13333333333333333
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "_QueryModel(filter='python')",
+      "got": "{'name':",
+      "got_raw": "{'name': 'John'}",
+      "exact_match": false,
+      "code_bleu": 0.18995892141289814,
+      "edit_similarity": 0.16666666666666666
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "_HeaderModel(whatever='yes')",
+      "got": "{'X-User':",
+      "got_raw": "{'X-User': '1'}",
+      "exact_match": false,
+      "code_bleu": 0.18995892141289814,
+      "edit_similarity": 0.15789473684210525
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "dataclasses.asdict(decoded_token)",
+      "got": "dataclasses.asdict(decoded_token)",
+      "got_raw": "dataclasses.asdict(decoded_token)",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot({'format': 'invalid-format'})",
+      "got": "{}",
+      "got_raw": "{}",
+      "exact_match": false,
+      "code_bleu": 0.14865088937534013,
+      "edit_similarity": 0.1
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "response.content.decode('utf-8').lower()",
+      "got": "response.content.decode()",
+      "got_raw": "response.content.decode()",
+      "exact_match": false,
+      "code_bleu": 0.5271011687724095,
+      "edit_similarity": 0.7692307692307693
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "EndpointMetadataError, match=str(status))",
+      "got": "EndpointMetadataError)",
+      "got_raw": "EndpointMetadataError)",
+      "exact_match": false,
+      "code_bleu": 0.1321341238891912,
+      "edit_similarity": 0.6984126984126984
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot( {'message': 'Page not found'}, )",
+      "got": "{'message': 'Not found'}",
+      "got_raw": "{'message': 'Not found'}",
+      "exact_match": false,
+      "code_bleu": 0.2984916477287617,
+      "edit_similarity": 0.696969696969697
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot({ 'email': email, 'age': user_id, })",
+      "got": "{ 'email': email, 'age': faker.unique.random_int(), }",
+      "got_raw": "{ 'email': email, 'age': faker.unique.random_int(), }",
+      "exact_match": false,
+      "code_bleu": 0.4738611152748619,
+      "edit_similarity": 0.6530612244897959
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot({ 'django_session': [], 'csrf': [], })",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.09428090415820634,
+      "edit_similarity": 0.08163265306122448
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot({ 'detail': [{'msg': 'API Error'}], })",
+      "got": "{ 'error': 'API Error', 'status_code': 409, }",
+      "got_raw": "{ 'error': 'API Error', 'status_code': 409, }",
+      "exact_match": false,
+      "code_bleu": 0.15128026243358708,
+      "edit_similarity": 0.45652173913043476
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot( {'lax_field': 1, 'strict_field': 1}, )",
+      "got": "{'lax_field': '1', 'strict_field': 1}",
+      "got_raw": "{'lax_field': '1', 'strict_field': 1}",
+      "exact_match": false,
+      "code_bleu": 0.45556408140663196,
+      "edit_similarity": 0.8235294117647058
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot({ 'error': [{'message': 'test msg'}], })",
+      "got": "{ 'error': 'Custom error message', }",
+      "got_raw": "{ 'error': 'Custom error message', }",
+      "exact_match": false,
+      "code_bleu": 0.1955692089739604,
+      "edit_similarity": 0.5411764705882353
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot( {'first_field': 1, 'second_field': 1}, )",
+      "got": "{ 'first_field': '1', 'second_field': 1, }",
+      "got_raw": "{ 'first_field': '1', 'second_field': 1, }",
+      "exact_match": false,
+      "code_bleu": 0.4194207015100664,
+      "edit_similarity": 0.8260869565217391
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot({ 'detail': [{'msg': 'Internal server error'}], })",
+      "got": "{ 'error': 'Error: Unknown type: <class \\'str\\'>', }",
+      "got_raw": "{ 'error': 'Error: Unknown type: <class \\'str\\'>', }",
+      "exact_match": false,
+      "code_bleu": 0.13212657272055228,
+      "edit_similarity": 0.21621621621621623
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot({ 'detail': [{'msg': 'Internal server error'}], })",
+      "got": "{ 'error': 'Not serializable', }",
+      "got_raw": "{ 'error': 'Not serializable', }",
+      "exact_match": false,
+      "code_bleu": 0.13212657272055228,
+      "edit_similarity": 0.26373626373626374
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot( 'Set-Cookie: error_id=value; Path=/; SameSite=lax', )",
+      "got": "{ \"dmr-api-error\": \"1\", }",
+      "got_raw": "{ \"dmr-api-error\": \"1\", }",
+      "exact_match": false,
+      "code_bleu": 0.22089591134157885,
+      "edit_similarity": 0.25
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot( 'Set-Cookie: optional=123; Path=/; SameSite=strict', )",
+      "got": "[]",
+      "got_raw": "[]",
+      "exact_match": false,
+      "code_bleu": 0.18072040072196896,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot({ 'detail': [{'msg': IsStr(), 'type': 'not_allowed'}], })",
+      "got": "{ 'error': 'Method not allowed', 'status': 405, }",
+      "got_raw": "{ 'error': 'Method not allowed', 'status': 405, }",
+      "exact_match": false,
+      "code_bleu": 0.09764082017258031,
+      "edit_similarity": 0.4
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "snapshot( {'detail': [{'msg': 'Page not found', 'type': 'not_found'}]}, )",
+      "got": "{ 'error': 'Not found', 'error_type': ErrorType.NOT_FOUND, }",
+      "got_raw": "{ 'error': 'Not found', 'error_type': ErrorType.NOT_FOUND, }",
+      "exact_match": false,
+      "code_bleu": 0.0897579312203396,
+      "edit_similarity": 0.45112781954887216
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "1",
+      "got": "1",
+      "got_raw": "1",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "1",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "1",
+      "got": "{",
+      "got_raw": "{ 'status': 1, }",
+      "exact_match": false,
+      "code_bleu": 0.5,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "0",
+      "got": "0",
+      "got_raw": "0",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "1",
+      "got": "{'test':",
+      "got_raw": "{'test': 'foo'}",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "1",
+      "got": "[1]",
+      "got_raw": "[1]",
+      "exact_match": false,
+      "code_bleu": 0.4518010018049224,
+      "edit_similarity": 0.5
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "1",
+      "got": "{'foo':",
+      "got_raw": "{'foo': 'bar'}",
+      "exact_match": false,
+      "code_bleu": 0.37991784282579627,
+      "edit_similarity": 0.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "2",
+      "got": "2",
+      "got_raw": "2",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "3",
+      "got": "3",
+      "got_raw": "3",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    },
+    {
+      "repo": "wemake-services/django-modern-rest",
+      "expected": "True",
+      "got": "True",
+      "got_raw": "True",
+      "exact_match": true,
+      "code_bleu": 1.0,
+      "edit_similarity": 1.0
+    }
+  ]
+}
\ No newline at end of file