diff --git "a/hypernet/scale_500repos_results/cr_test/results.json" "b/hypernet/scale_500repos_results/cr_test/results.json" new file mode 100644--- /dev/null +++ "b/hypernet/scale_500repos_results/cr_test/results.json" @@ -0,0 +1,57736 @@ +{ + "exact_match_pct": 61.17867165575304, + "exact_match_count": 3924, + "n": 6414, + "n_total": 6414, + "code_bleu": 0.7587371974453063, + "edit_similarity": 0.7669861929240352, + "entries": [ + { + "repo": "15r10nk/inline-snapshot", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "real_repr", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "None", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "stderr", + "got": "stderr", + "got_raw": "stderr", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "report", + "got": "report", + "got_raw": "report", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "returncode", + "got": "returncode", + "got_raw": "returncode", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "f\"ERROR: {e}\\n\"", + "got": "e.stderr", + "got_raw": "e.stderr", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.08695652173913043 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "reported_categories", + "got": "reported_categories", + "got_raw": "reported_categories", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "self._changed_files(tmp_path)", + "got": "self._changed_files(tmp_path)", + "got_raw": "self._changed_files(tmp_path)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "normalize(report_output.getvalue())", + "got": "self._report(tmp_path)", + "got_raw": "self._report(tmp_path)", + "exact_match": false, + "code_bleu": 0.19701824208181185, + "edit_similarity": 0.3157894736842105 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "parse_outcomes(result_stdout.splitlines())", + "got": "parse_outcomes(result_stdout.splitlines())", + "got_raw": "parse_outcomes(result_stdout.splitlines())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "num_args", + "got": "num_args", + "got_raw": "num_args", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "number", + "got": "number", + "got_raw": "number", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "self.header", + "got": "self.header", + "got_raw": "self.header", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "reported_flags", + "got": "flags", + "got_raw": "flags", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5263157894736842 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "\"fix\" in s.flags", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"mxxm\")", + "got": "snapshot(\"mddiimx\")", + "got_raw": "snapshot(\"mddiimx\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"dmmmi\")", + "got": "snapshot({\"iabc\":", + "got_raw": "snapshot({\"iabc\": \"iabc\", \"abcd\": \"abcd\"})", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"mddiim\")", + "got": "snapshot(\"xxyy\")", + "got_raw": "snapshot(\"xxyy\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "new_code", + "got": "new_code", + "got_raw": "new_code", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "eval(code)", + "got": "eval(code)", + "got_raw": "eval(code)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"B()\")", + "got": "snapshot(\"B(b=0)\")", + "got_raw": "snapshot(\"B(b=0)\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "HasRepr(int, \"5\")", + "got": "5", + "got_raw": "5", + "exact_match": false, + "code_bleu": 0.08333333333333333, + "edit_similarity": 0.1111111111111111 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"B(b=5)\")", + "got": "snapshot(\"B(b=5)\")", + "got_raw": "snapshot(\"B(b=5)\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"C(a=1, c=2)\")", + "got": "\"{'a': 1, 'c': 2}\"", + "got_raw": "\"{'a': 1, 'c': 2}\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.3902439024390244 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"FakeTuple()\")", + "got": "\"FakeTuple()\"", + "got_raw": "\"FakeTuple()\"", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.7222222222222222 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"Dataclass(a=0)\")", + "got": "snapshot(\"Dataclass(a=0", + "got_raw": "snapshot(\"Dataclass(a=0, b=0, c=[])\")", + "exact_match": false, + "code_bleu": 0.21105340631872638, + "edit_similarity": 0.9387755102040817 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"{'a', 'b', 1, 2, 3}\")", + "got": "\"{1, 2, 3, 'a', True, 'b'}\"", + "got_raw": "\"{1, 2, 3, 'a', True, 'b'}\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.3793103448275862 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"Counter({1: 3, 2: 1})\")", + "got": "snapshot(\"Counter([1, 1, 1, 2])\")", + "got_raw": "snapshot(\"Counter([1, 1, 1, 2])\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7878787878787878 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"{1, 2, 3, 10, 11, 20, 200}\" )", + "got": "snapshot( \"{'200', '11', '20', '10', 200, 20, 11, 20, 3, 2, 1}\")", + "got_raw": "snapshot( \"{'200', '11', '20', '10', 200, 20, 11, 20, 3, 2, 1}\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6346153846153846 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"{'a', 'b', 1j, 2j, 3j, True}\" )", + "got": "snapshot(\"{'a', 'b', 1j, 2j, 3j}\")", + "got_raw": "snapshot(\"{'a', 'b', 1j, 2j, 3j}\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8947368421052632 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ class Color: pass assert [Color,int] == snapshot([Color, int]) \"\"\" )", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.08333333333333333, + "edit_similarity": 0.04597701149425287 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ from enum import Flag, auto class Color(Flag): red = auto() green = auto() blue = auto() assert Color.red | Color.blue == snapshot(Color.red | Color.blue) \"\"\" )", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.08333333333333333, + "edit_similarity": 0.0335195530726257 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ from enum import Enum from dataclasses import dataclass class color(Enum): red=\"red\" blue=\"blue\" @dataclass class container: bg: color=color.red fg: color=color.blue assert container(bg=color.red,fg=color.red) == snapshot(container(fg=color.red)) \"\"\" )", + "got": "\"Enum\"", + "got_raw": "\"Enum\"", + "exact_match": false, + "code_bleu": 0.05, + "edit_similarity": 0.029304029304029304 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "blocks", + "got": "blocks", + "got_raw": "blocks", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "new_code", + "got": "new_code", + "got_raw": "new_code", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "recorded_markdown_code", + "got": "recorded_markdown_code", + "got_raw": "recorded_markdown_code", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "5", + "got": "\"5\"", + "got_raw": "\"5\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "[0, 5]", + "got": "snapshot([0, external(\"hash:ef2d127de37b*.json\")])", + "got_raw": "snapshot([0, external(\"hash:ef2d127de37b*.json\")])", + "exact_match": false, + "code_bleu": 0.22416933501922287, + "edit_similarity": 0.17857142857142858 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"5\")", + "got": "\"5\"", + "got_raw": "\"5\"", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.375 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"[0, 5]\")", + "got": "[0, 5]", + "got_raw": "[0, 5]", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.5 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "A(a=outsource(5), b=2)", + "got": "snapshot(A(a=external(\"hash:ef2d127de37b*.json\"), b=2))", + "got_raw": "snapshot(A(a=external(\"hash:ef2d127de37b*.json\"), b=2))", + "exact_match": false, + "code_bleu": 0.5071960939456881, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"A(a=5, b=2)\")", + "got": "\"A(a=5, b=2)\"", + "got_raw": "\"A(a=5, b=2)\"", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.7222222222222222 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "HasRepr(float, \"nan\")", + "got": "float(\"nan\")", + "got_raw": "float(\"nan\")", + "exact_match": false, + "code_bleu": 0.35818997727451396, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "HasRepr(Thing, \"\")", + "got": "HasRepr(Thing, \"\")", + "got_raw": "HasRepr(Thing, \"\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "code", + "got": "code", + "got_raw": "code", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "set()", + "got": "set(all_flags)", + "got_raw": "set(all_flags)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.5263157894736842 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "test2", + "got": "test2", + "got_raw": "test2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "s.error", + "got": "s.fix", + "got_raw": "s.fix", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "s.source", + "got": "code", + "got_raw": "code", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "s2.source", + "got": "s2.source", + "got_raw": "s2.source", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot()", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(1)", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"\")", + "got": "snapshot(", + "got_raw": "snapshot( \"\"\"\\\n------------------------------- Report changes -------------------------------\n+----------------------------- tests/test_file.py -----------------------------+\n| @@ -19,4 +19,9 @@ |\n| |\n| |\n| |\n| def test_thing(): |\n| - assert X(\"a\" * 40, \"a\" * 40) == snapshot() |\n| + assert X(\"a\" * 40, \"a\" * 40) == snapshot( |\n| + X( |\n| + \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\", |\n| + \"", + "exact_match": false, + "code_bleu": 0.3535533905932738, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "\"snapshot()\\n\"", + "got": "\"assert", + "got_raw": "\"assert 1 + 2\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.38095238095238093 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "AssertionError)", + "got": "AssertionError)", + "got_raw": "AssertionError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "{reported_flag}", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.30120066786994826, + "edit_similarity": 0.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(set())", + "got": "snapshot()", + "got_raw": "snapshot()", + "exact_match": false, + "code_bleu": 0.3535533905932738, + "edit_similarity": 0.8 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "str(error.value)", + "got": "error.value.args[0]", + "got_raw": "error.value.args[0]", + "exact_match": false, + "code_bleu": 0.2984745896009823, + "edit_similarity": 0.6285714285714286 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "expected_warnings", + "got": "expected_warnings", + "got_raw": "expected_warnings", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "\"Flags({'update'})\"", + "got": "\"Flags({'update'})\"", + "got_raw": "\"Flags({'update'})\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot({\"create\"})", + "got": "Flags()", + "got_raw": "Flags()", + "exact_match": false, + "code_bleu": 0.24999999999999997, + "edit_similarity": 0.2962962962962963 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "repr(Is(5))", + "got": "repr(Is(5))", + "got_raw": "repr(Is(5))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "flags", + "got": "all_flags", + "got_raw": "all_flags", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "s1.error", + "got": "\"fix\"", + "got_raw": "\"fix\" in s1", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "new_code", + "got": "new_code", + "got_raw": "new_code", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"assert {}==snapshot({})\")", + "got": "snapshot(\"assert {1:1}==snapshot({1:0+1})\")", + "got_raw": "snapshot(\"assert {1:1}==snapshot({1:0+1})\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8974358974358975 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"assert [1,2]==snapshot([0+1,2])\")", + "got": "[1, 2, 3]", + "got_raw": "[1, 2, 3]", + "exact_match": false, + "code_bleu": 0.1561969968460128, + "edit_similarity": 0.23076923076923078 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"assert {1:1}==snapshot({1:0+1})\")", + "got": "{\"a\": 1", + "got_raw": "{\"a\": 1, \"b\": 2}", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.12 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"assert [1,5]==snapshot([0+1, 5])\")", + "got": "[1, 2, 3]", + "got_raw": "[1, 2, 3]", + "exact_match": false, + "code_bleu": 0.1561969968460128, + "edit_similarity": 0.22641509433962265 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"assert (1,5)==snapshot((0+1, 5))\")", + "got": "((1, 2)", + "got_raw": "((1, 2)", + "exact_match": false, + "code_bleu": 0.24446151121745047, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot('assert {1+2:\"3\"}==snapshot({1+2:\"3\"})')", + "got": "{\"a\": 1}", + "got_raw": "{\"a\": 1}", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.17543859649122806 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"assert {1:1, 2:2}==snapshot({1:0+1, 2:2})\")", + "got": "{\"a\": 1}", + "got_raw": "{\"a\": 1}", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.09836065573770492 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"assert [1,2,3,4,5,6]==snapshot([0+1, 2, 3, 4, 5, 6])\")", + "got": "[1, 2, 3]", + "got_raw": "[1, 2, 3]", + "exact_match": false, + "code_bleu": 0.1561969968460128, + "edit_similarity": 0.2465753424657534 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( 'assert {0:\"before\",1:1,2:\"after\"}==snapshot({0: \"before\", 1:0+1, 2: \"after\"})' )", + "got": "{\"a\": 1, \"b\": 2}", + "got_raw": "{\"a\": 1, \"b\": 2}", + "exact_match": false, + "code_bleu": 0.11868405219520975, + "edit_similarity": 0.22429906542056074 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ from dirty_equals import IsInt assert {5:5,2:2}==snapshot({5:IsInt(),2:2}) \"\"\" )", + "got": "snapshot({\"a\": \"a\"})", + "got_raw": "snapshot({\"a\": \"a\"})", + "exact_match": false, + "code_bleu": 0.22679164443904004, + "edit_similarity": 0.2608695652173913 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ left = { \"a\": 1, \"b\": { \"c\": 2, \"d\": [ 3, 4, 5, ], }, \"e\": ( { \"f\": 6, \"g\": 7, }, ), } assert left == snapshot( { \"a\": 1, \"b\": { \"c\": 2 * 1 + 0, \"d\": [ int(3), 4, 5, ], }, \"e\": ({\"f\": 3 + 3, \"g\": 7},), } ) \"\"\" )", + "got": "snapshot({\"a\": \"A\"})", + "got_raw": "snapshot({\"a\": \"A\"})", + "exact_match": false, + "code_bleu": 0.05336273986800942, + "edit_similarity": 0.07317073170731707 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"\")", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.2259005009024612, + "edit_similarity": 0.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ def test_a(): assert 5==snapshot(5) \"\"\" )", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.09910059291689341, + "edit_similarity": 0.06896551724137931 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ def test_a(): assert 4==snapshot(4) \"\"\" )", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.09910059291689341, + "edit_similarity": 0.06896551724137931 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ def test_a(): assert 5 in snapshot([5]) \"\"\" )", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.09910059291689341, + "edit_similarity": 0.06451612903225806 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ E assert 2 == 1 E + where 1 = snapshot() \"\"\" )", + "got": "[ \"\"\"\\", + "got_raw": "[ \"\"\"\\\nfor e in (1, 2):", + "exact_match": false, + "code_bleu": 0.23953607464205617, + "edit_similarity": 0.14925373134328357 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ def test_a(): assert \"5\" == snapshot(\"5\") \"\"\" )", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.05946035575013606, + "edit_similarity": 0.0625 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ E assert 2 == 1 E + where 1 = snapshot(1) \"\"\" )", + "got": "[ \"\"\"\\", + "got_raw": "[ \"\"\"\\\nAssertionError: 1 != 1", + "exact_match": false, + "code_bleu": 0.23953607464205617, + "edit_similarity": 0.14705882352941177 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ def test_something(): assert 2 == snapshot(2) \"\"\" )", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.09910059291689341, + "edit_similarity": 0.058823529411764705 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"ERROR: --inline-snapshot=disable can not be combined with other flags (fix)\" )", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.04395604395604396 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"ERROR: --inline-snapshot=disable can not be combined with other flags (review)\" )", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.0425531914893617 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ > assert \"a\"==snapshot(\"b\") E AssertionError: assert 'a' == 'b' E \\n\\ E - b E + a \"\"\" )", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.09036020036098448, + "edit_similarity": 0.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ > assert snapshot(\"b\")==\"a\" E AssertionError: assert 'b' == 'a' E \\n\\ E - a E + b \"\"\" )", + "got": "snapshot( \"\"\"\\", + "got_raw": "snapshot( \"\"\"\\\n> assert snapshot(\"b\")==\"a\"", + "exact_match": false, + "code_bleu": 0.25406637407730737, + "edit_similarity": 0.2413793103448276 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ def test_a(): assert list(range(10)) == snapshot( [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] ) \"\"\" )", + "got": "\"\"\"\"\"\"", + "got_raw": "\"\"\"\"\"\"", + "exact_match": false, + "code_bleu": 0.24999999999999997, + "edit_similarity": 0.11009174311926606 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ def test_a(): assert \"5\" == snapshot('''5''') assert 5 <= snapshot(5) assert 5 == snapshot(5) \"\"\" )", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.07432544468767006, + "edit_similarity": 0.034482758620689655 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ def test_a(): assert \"5\" == snapshot('''5''') assert 5 <= snapshot(5) assert 5 == snapshot(4) \"\"\" )", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.07432544468767006, + "edit_similarity": 0.034482758620689655 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "b", + "got": "b", + "got_raw": "b", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "a", + "got": "a", + "got_raw": "a", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "r", + "got": "r", + "got_raw": "r", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "1", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "ValueError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "\"\"\" 12a45 1245 12c345 \"\"\"", + "got": "b\"\"\"a", + "got_raw": "b\"\"\"a\na\na\nc\"\"\"", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "s", + "got": "snapshot({\"s\":", + "got_raw": "snapshot({\"s\": s})", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"s=snapshot('\\\\n')\")", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.12903225806451613 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot('s=snapshot(\"a\\\\n\")')", + "got": "\"abc\"", + "got_raw": "\"abc\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.17142857142857143 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"s=snapshot('abc\\\\n')\")", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "'assert \"ab\" == snapshot(\"a\" \"b\")'", + "got": "snapshot({\"a\": \"a\"", + "got_raw": "snapshot({\"a\": \"a\", \"b\": \"b\", \"c\": \"c\"})", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.5769230769230769 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot('s = snapshot(b\"a\\\\nb\")')", + "got": "snapshot('''\\", + "got_raw": "snapshot('''\\", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.46808510638297873 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "'assert \"ab\" == snapshot(\"a\"\\n \"b\")'", + "got": "\"assert \\\"ab\\\" == snapshot(\\\"a\\\" \\\"b\\\")\"", + "got_raw": "\"assert \\\"ab\\\" == snapshot(\\\"a\\\" \\\"b\\\")\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8421052631578947 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "'assert b\"ab\" == snapshot(b\"a\"\\n b\"b\")'", + "got": "check_update('assert \"ab\" == snapshot(\"a\" \"b\")')", + "got_raw": "check_update('assert \"ab\" == snapshot(\"a\" \"b\")')", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.7816091954022989 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( '''\\ s=snapshot(\"\"\"\\\\ a a \"\"\")\\ ''' )", + "got": "snapshot(", + "got_raw": "snapshot(", + "exact_match": false, + "code_bleu": 0.23570226039551584, + "edit_similarity": 0.32142857142857145 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( '''\\ s = snapshot(\"\"\"\\\\ '\\\\ \"\"\")\\ ''' )", + "got": "snapshot(\"\"\"\\", + "got_raw": "snapshot(\"\"\"\\", + "exact_match": false, + "code_bleu": 0.18187338634676456, + "edit_similarity": 0.41935483870967744 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( '''\\ s = snapshot(\"\"\"\\\\ \"\\\\ \"\"\")\\ ''' )", + "got": "snapshot('''\\", + "got_raw": "snapshot('''\\\ns = snapshot(\"\"\"\\\\\n\"\\\\\n\"\"\")\\", + "exact_match": false, + "code_bleu": 0.4234439567955123, + "edit_similarity": 0.41935483870967744 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( '''\\ s = snapshot(\"\"\"\\\\ a b\\\\ \"\"\")\\ ''' )", + "got": "snapshot(\"a\\nb\")", + "got_raw": "snapshot(\"a\\nb\")", + "exact_match": false, + "code_bleu": 0.3387551654364098, + "edit_similarity": 0.417910447761194 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( '''\\ s = snapshot(\"\"\"\\\\ \\\\x08\\\\ \"\"\")\\ ''' )", + "got": "snapshot('''\\", + "got_raw": "snapshot('''\\", + "exact_match": false, + "code_bleu": 0.4234439567955123, + "edit_similarity": 0.3939393939393939 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( '''\\ assert \"ab\\\\nc\" == snapshot(\"\"\"\\\\ ab c\\\\ \"\"\")\\ ''' )", + "got": "\"ab\"", + "got_raw": "\"ab\"", + "exact_match": false, + "code_bleu": 0.08333333333333333, + "edit_similarity": 0.11267605633802817 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "4", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"\")", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot([])", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.28284271247461906, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"ERROR: --inline-snapshot=create can not be combined with xdist\" )", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.05128205128205128 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ Error: one snapshot has incorrect values (--inline-snapshot=fix) You can also use --inline-snapshot=review to approve the changes interactively \"\"\" )", + "got": "snapshot({ \"errors\": [ { \"name\": \"test_a\"", + "got_raw": "snapshot({ \"errors\": [ { \"name\": \"test_a\", \"outcome\": \"error\", \"result\": snapshot( { \"error\": \"AssertionError\", \"message\": \"1==2\", \"stack\": \"AssertionError: 1==2\", } ), } ], \"failures\": [ { \"name\": \"test_a\", \"outcome\": \"failure\", \"result\": snapshot( { \"error\": \"AssertionError\", \"message\": \"1==2\", \"stack\": \"AssertionError: 1==2\", } ), } ], })", + "exact_match": false, + "code_bleu": 0.16590387014219712, + "edit_similarity": 0.2146341463414634 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot(\"\")", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot([])", + "got": "snapshot()", + "got_raw": "snapshot()", + "exact_match": false, + "code_bleu": 0.38336586254776345, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "project.source", + "got": "project.source", + "got_raw": "project.source", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "outsource(\"one\")", + "got": "\"one\"", + "got_raw": "\"one\"", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "outsource(\"two\")", + "got": "outsource(\"two\")", + "got_raw": "outsource(\"two\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "external(\"hash:bbbbb*.txt\")", + "got": "\"test4\"", + "got_raw": "\"test4\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.29411764705882354 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ from os import environ", + "got": "b\"\"", + "got_raw": "b\"\"", + "exact_match": false, + "code_bleu": 0.11939665909150465, + "edit_similarity": 0.1 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( [\"9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08.txt\"] )", + "got": "snapshot([])", + "got_raw": "snapshot([])", + "exact_match": false, + "code_bleu": 0.503562566226128, + "edit_similarity": 0.25 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( [\"2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae.txt\"] )", + "got": "snapshot()", + "got_raw": "snapshot()", + "exact_match": false, + "code_bleu": 0.3194715521231362, + "edit_similarity": 0.2127659574468085 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ from os import environ from os import getcwd from os import chdir \"\"\" )", + "got": "b\"\"", + "got_raw": "b\"\"", + "exact_match": false, + "code_bleu": 0.17909498863725698, + "edit_similarity": 0.0449438202247191 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ ''' docstring ''' from __future__ import annotations from os import chdir \"\"\" )", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.09910059291689341, + "edit_similarity": 0.041666666666666664 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"assert outsource('text') == snapshot(external(\\\"uuid:f728b4fa-4248-4e3a-8a5d-2f346baa9455.txt\\\"))\" )", + "got": "snapshot({\"a\": 1})", + "got_raw": "snapshot({\"a\": 1})", + "exact_match": false, + "code_bleu": 0.22679164443904004, + "edit_similarity": 0.23255813953488372 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( [ ExternalLocation( storage=\"hash\", stem=\"111*\", suffix=\".txt\", filename=Path(\"a.py\"), linenumber=1, ) ] )", + "got": "snapshot({\"a\": \"b\"})", + "got_raw": "snapshot({\"a\": \"b\"})", + "exact_match": false, + "code_bleu": 0.058526875984268395, + "edit_similarity": 0.22058823529411764 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ > assert outsource(\"test2\") == s E AssertionError: assert 'test2' == 'test' E \\n\\ E - test E + test2 E ? + \"\"\" )", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.1129502504512306, + "edit_similarity": 0.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( \"\"\"\\ from inline_snapshot import external def test_a(): assert outsource(\"test\") == snapshot(external(\"hash:9f86d081884c*.txt\")) \"\"\" )", + "got": "default_result", + "got_raw": "default_result", + "exact_match": false, + "code_bleu": 0.03125, + "edit_similarity": 0.12658227848101267 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "snapshot( [ \"60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752.txt\", \"9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08.txt\", ] )", + "got": "snapshot()", + "got_raw": "snapshot()", + "exact_match": false, + "code_bleu": 0.21298103474875746, + "edit_similarity": 0.11834319526627218 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "\"test\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "results[name]", + "got": "results[name]", + "got_raw": "results[name]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "15r10nk/inline-snapshot", + "expected": "f\"{type(e).__name__}: {e}\"", + "got": "e", + "got_raw": "e", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.07407407407407407 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "3", + "got": "26", + "got_raw": "26", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "name2", + "got": "name2", + "got_raw": "name2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "name1", + "got": "name1", + "got_raw": "name1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "LANG_TO_INDEX", + "got": "LANG_TO_INDEX", + "got_raw": "LANG_TO_INDEX", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "skip", + "got": "skip", + "got_raw": "skip", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "False", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "source", + "got": "source", + "got_raw": "source", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "\"reference\"", + "got": "\"reference\"", + "got_raw": "\"reference\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "field_names", + "got": "field_names", + "got_raw": "field_names", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "\"persona_no_ge\"", + "got": "\"persona_no_ge\"", + "got_raw": "\"persona_no_ge\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "\"persona_with_ge\"", + "got": "\"persona_with_ge\"", + "got_raw": "\"persona_with_ge\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "ctx.get_skip_components()", + "got": "ctx", + "got_raw": "ctx", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.21428571428571427 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "\"ja\"", + "got": "\"ja\"", + "got_raw": "\"ja\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "\"en\"", + "got": "\"en\"", + "got_raw": "\"en\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "\"zh\"", + "got": "\"zh\"", + "got_raw": "\"zh\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "config", + "got": "config", + "got_raw": "config", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "'new.wav'", + "got": "\"new.wav\"", + "got_raw": "\"new.wav\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "ext.lower()", + "got": "ext.lower()", + "got_raw": "ext.lower()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "SUPPORTED_AUDIO_EXTS", + "got": "SUPPORTED_AUDIO_EXTS", + "got_raw": "SUPPORTED_AUDIO_EXTS", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "\"gpu\"", + "got": "\"gpu\"", + "got_raw": "\"gpu\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "EnvironmentMismatchError)", + "got": "EnvironmentMismatchError)", + "got_raw": "EnvironmentMismatchError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "EnvironmentStatus.CPU_ONLY", + "got": "EnvironmentStatus.CPU_ONLY", + "got_raw": "EnvironmentStatus.CPU_ONLY", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "EnvironmentStatus.GPU_READY", + "got": "EnvironmentStatus.GPU_READY", + "got_raw": "EnvironmentStatus.GPU_READY", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "EnvironmentStatus.GPU_DEPS_MISSING", + "got": "EnvironmentStatus.DEPENDENCIES_MISSING", + "got_raw": "EnvironmentStatus.DEPENDENCIES_MISSING", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "frontend2", + "got": "frontend2", + "got_raw": "frontend2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "frontend3", + "got": "frontend3", + "got_raw": "frontend3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "languages", + "got": "languages", + "got_raw": "languages", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "mock_frontend", + "got": "mock_frontend", + "got_raw": "mock_frontend", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "mock_instance_2", + "got": "mock_instance_2", + "got_raw": "mock_instance_2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "ValueError, match=\"Unsupported language\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.4230769230769231 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "mgr2", + "got": "mgr2", + "got_raw": "mgr2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "status", + "got": "status", + "got_raw": "status", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "\"v2\"", + "got": "\"v2\"", + "got_raw": "\"v2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "inputs", + "got": "inputs", + "got_raw": "inputs", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "\"v2Pro\"", + "got": "\"v2-pro\"", + "got_raw": "\"v2-pro\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "V2_SPEC", + "got": "V2_SPEC", + "got_raw": "V2_SPEC", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "\"v2ProPlus\"", + "got": "\"v2ProPlus\"", + "got_raw": "\"v2ProPlus\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "V2_PRO_SPEC", + "got": "V2_PRO_SPEC", + "got_raw": "V2_PRO_SPEC", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "[\"a\", \"b\", \"c\"]", + "got": "[\"a\", \"b\", \"c\"]", + "got_raw": "[\"a\", \"b\", \"c\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "V2_PRO_PLUS_SPEC", + "got": "V2_PRO_PLUS_SPEC", + "got_raw": "V2_PRO_PLUS_SPEC", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "source", + "got": "source", + "got_raw": "source", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "skip_components", + "got": "skip_components", + "got_raw": "skip_components", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "first_call_count", + "got": "first_call_count", + "got_raw": "first_call_count", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lux-Luna/LunaVox", + "expected": "kwargs.get(\"skip_components\", set())", + "got": "kwargs", + "got_raw": "kwargs", + "exact_match": false, + "code_bleu": 0.05946035575013606, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "MartenBE/mkslides", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MartenBE/mkslides", + "expected": "content", + "got": "content", + "got_raw": "content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MartenBE/mkslides", + "expected": "output_path / \"index.html\")", + "got": "output_path / \"mkslides-assets/highlight-js-themes/monokai.js\")", + "got_raw": "output_path / \"mkslides-assets/highlight-js-themes/monokai.js\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.5111111111111111 + }, + { + "repo": "MartenBE/mkslides", + "expected": "output_path / \"randomfile-1.txt\")", + "got": "output_path / \"someslides-1.md\")", + "got_raw": "output_path / \"someslides-1.md\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7076923076923077 + }, + { + "repo": "MartenBE/mkslides", + "expected": "output_path / \"someslides-1.html\")", + "got": "output_path / \"slides.html\")", + "got_raw": "output_path / \"slides.html\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.9032258064516129 + }, + { + "repo": "MartenBE/mkslides", + "expected": "output_path / \"extra/randomfile-3.txt\")", + "got": "output_path / \"someslides-1.md\")", + "got_raw": "output_path / \"someslides-1.md\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6197183098591549 + }, + { + "repo": "MartenBE/mkslides", + "expected": "output_path / \"somefolder/randomfile-2.txt\")", + "got": "output_path / \"randomfile-2.txt\")", + "got_raw": "output_path / \"randomfile-2.txt\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"../assets/favicon.ico\")", + "got_raw": "\"../assets/favicon.ico\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"favicon.png\"", + "got_raw": "\"favicon.png\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.3389830508474576 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"assets/favicon.ico\")", + "got_raw": "\"assets/favicon.ico\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"favicon-3.ico\")", + "got_raw": "\"favicon-3.ico\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.43478260869565216 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"favicon.png\"", + "got_raw": "\"favicon.png\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.21739130434782608 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"assets/highlight-theme.css\")", + "got_raw": "\"assets/highlight-theme.css\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.22857142857142856 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"frontmatter\"", + "got_raw": "\"frontmatter\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.2033898305084746 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"favicon-2.ico\")", + "got_raw": "\"favicon-2.ico\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.25396825396825395 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"assets\"", + "got_raw": "\"assets\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.2807017543859649 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'frontmatter title')", + "got": "\"slides\"", + "got_raw": "\"slides\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"assets/favicon.ico\")", + "got_raw": "\"assets/favicon.ico\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3013698630136986 + }, + { + "repo": "MartenBE/mkslides", + "expected": "output_path / \"mkslides-assets/highlight-js-themes/vs.css\")", + "got": "output_path / \"mkslides-assets/reveal-js/dist/theme/solarized-light.css\")", + "got_raw": "output_path / \"mkslides-assets/reveal-js/dist/theme/solarized-light.css\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "MartenBE/mkslides", + "expected": "output_path / \"mkslides-assets/reveal-js/dist/theme/solarized.css\")", + "got": "\"frontmatter title\")", + "got_raw": "\"frontmatter title\")", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.25287356321839083 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"mkslides-assets/highlight-js-themes/vs.css\")", + "got_raw": "\"mkslides-assets/highlight-js-themes/vs.css\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7317073170731707 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"theme\"", + "got_raw": "\"theme\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.15053763440860216 + }, + { + "repo": "MartenBE/mkslides", + "expected": "output_path / \"example-4.png\")", + "got": "output_path / \"img/somefolder/example-4.png\")", + "got_raw": "output_path / \"img/somefolder/example-4.png\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8 + }, + { + "repo": "MartenBE/mkslides", + "expected": "output_path / \"img/example-1.png\")", + "got": "output_path / \"slides\")", + "got_raw": "output_path / \"slides\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "MartenBE/mkslides", + "expected": "output_path / \"somefolder/example-2.png\")", + "got": "output_path / \"img/example-2.png\")", + "got_raw": "output_path / \"img/example-2.png\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8533333333333334 + }, + { + "repo": "MartenBE/mkslides", + "expected": "output_path / \"img/somefolder/example-3.png\")", + "got": "output_path / \"img/example-3.png\")", + "got_raw": "output_path / \"img/example-3.png\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8607594936708861 + }, + { + "repo": "MartenBE/mkslides", + "expected": "\"

Lorem ipsum

\")", + "got": "\"Lorem ipsum\")", + "got_raw": "\"Lorem ipsum\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7567567567567568 + }, + { + "repo": "MartenBE/mkslides", + "expected": "\"Lorem ipsum\")", + "got": "\"Index\")", + "got_raw": "\"Index\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.21621621621621623 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'Documentation built with MkSlides.')", + "got": "\"index\")", + "got_raw": "\"index\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.10714285714285714 + }, + { + "repo": "MartenBE/mkslides", + "expected": "re.compile( r\"\"\" \"\"\", re.VERBOSE | re.DOTALL, ))", + "got": "\"revealjs_markdown_data_options\"", + "got_raw": "\"revealjs_markdown_data_options\"", + "exact_match": false, + "code_bleu": 0.007575757575757576, + "edit_similarity": 0.08421052631578947 + }, + { + "repo": "MartenBE/mkslides", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MartenBE/mkslides", + "expected": "expected_returncode", + "got": "expected_returncode", + "got_raw": "expected_returncode", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'menu: {\"openButton\": true, \"openOnInit\": true}')", + "got": "\"menu\")", + "got_raw": "\"menu\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.21428571428571427 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"menu: {\"openButton\": true", + "got_raw": "\"menu: {\"openButton\": true, \"openOnInit\": true}\")", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.15789473684210525 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"plugins\"", + "got_raw": "\"plugins\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.09345794392523364 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"", + "got_raw": "\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.02 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"menu: {\"openButton\": true", + "got_raw": "\"menu: {\"openButton\": true, \"openOnInit\": true}\")", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "MartenBE/mkslides", + "expected": "re.compile( r\"\"\" aaaaa-aaaa .*? aaa-aa .*? a \"\"\", re.VERBOSE | re.DOTALL, ))", + "got": "\"slides\"", + "got_raw": "\"slides\"", + "exact_match": false, + "code_bleu": 0.02631578947368421, + "edit_similarity": 0.047619047619047616 + }, + { + "repo": "MartenBE/mkslides", + "expected": "re.compile( r\"\"\" bbbbb-bbbb .*? bbb-bb .*? b \"\"\", re.VERBOSE | re.DOTALL, ))", + "got": "\"aaaaa-aaaa\"", + "got_raw": "\"aaaaa-aaaa\"", + "exact_match": false, + "code_bleu": 0.02631578947368421, + "edit_similarity": 0.06818181818181818 + }, + { + "repo": "MartenBE/mkslides", + "expected": "expected_returncode", + "got": "expected_returncode", + "got_raw": "expected_returncode", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MartenBE/mkslides", + "expected": "\"[](/folder/test.md)\")", + "got": "\"", + "got_raw": "\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.08695652173913043 + }, + { + "repo": "MartenBE/mkslides", + "expected": "\"[](someslides-2.html)\")", + "got": "\"Slides\")", + "got_raw": "\"Slides\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.48484848484848486 + }, + { + "repo": "MartenBE/mkslides", + "expected": "\"[](./someslides-2.html)\")", + "got": "\"[someslides-2.html]\")", + "got_raw": "\"[someslides-2.html]\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.875 + }, + { + "repo": "MartenBE/mkslides", + "expected": "\"[test](/folder/test.md)\")", + "got": "\"test\")", + "got_raw": "\"test\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.42424242424242425 + }, + { + "repo": "MartenBE/mkslides", + "expected": "\"[test](someslides-2.html)\")", + "got": "\"[](../someslides-2.html)\")", + "got_raw": "\"[](../someslides-2.html)\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8727272727272727 + }, + { + "repo": "MartenBE/mkslides", + "expected": "\"history: true,\")", + "got": "\"
\")", + "got_raw": "\"
\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.28 + }, + { + "repo": "MartenBE/mkslides", + "expected": "re.compile( r\"\"\" Reveal\\.initialize\\({ .*? transition:\\s+\"fade\", .*? }\\); \"\"\", re.VERBOSE | re.DOTALL, ))", + "got": "\"revealjs\"", + "got_raw": "\"revealjs\"", + "exact_match": false, + "code_bleu": 0.023809523809523808, + "edit_similarity": 0.1391304347826087 + }, + { + "repo": "MartenBE/mkslides", + "expected": "re.compile( r\"\"\" Reveal\\.initialize\\({ .*? height:\\s+1080, .*? width:\\s+1920, .*? }\\); \"\"\", re.VERBOSE | re.DOTALL, ))", + "got": "\"revealjs_integer_options\"", + "got_raw": "\"revealjs_integer_options\"", + "exact_match": false, + "code_bleu": 0.02631578947368421, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "MartenBE/mkslides", + "expected": "\"

Index

\")", + "got": "\"Slides", + "got_raw": "\"Slides 2\")", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.25 + }, + { + "repo": "MartenBE/mkslides", + "expected": "output_path / \"index.html\")", + "got": "output_path / \"slides.html\")", + "got_raw": "output_path / \"slides.html\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "MartenBE/mkslides", + "expected": "\"Lorem ipsum dolor sit amet\")", + "got": "\"Hello world\")", + "got_raw": "\"Hello world\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.32558139534883723 + }, + { + "repo": "MartenBE/mkslides", + "expected": "output_path / \"someslides-1.html\")", + "got": "output_path / \"index.md\")", + "got_raw": "output_path / \"index.md\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7457627118644068 + }, + { + "repo": "MartenBE/mkslides", + "expected": "output_path / \"someslides-2.html\")", + "got": "output_path / \"someslides-2.html\")", + "got_raw": "output_path / \"someslides-2.html\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MartenBE/mkslides", + "expected": "\"\ud83d\udc4d\")", + "got": "\"\u26a0\ufe0f\")", + "got_raw": "\"\u26a0\ufe0f\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "MartenBE/mkslides", + "expected": "\"\u26a0\ufe0f\")", + "got": "\"Hello", + "got_raw": "\"Hello :earth_americas:\")", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"../assets/theme.css\")", + "got_raw": "\"../assets/theme.css\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.463768115942029 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"mkslides-assets/reveal-js/dist/reveal.css\")", + "got_raw": "\"mkslides-assets/reveal-js/dist/reveal.css\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3958333333333333 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "output_path / \"somefolder/someslides-2.html\")", + "got_raw": "output_path / \"somefolder/someslides-2.html\")", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.16 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "False)", + "got_raw": "False)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.12698412698412698 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"theme-4.css\"", + "got_raw": "\"theme-4.css\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "MartenBE/mkslides", + "expected": "output_path / \"mkslides-assets/reveal-js/dist/theme/simple.css\")", + "got": "output_path / \"index.html\")", + "got_raw": "output_path / \"index.html\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"mkslides-assets/reveal-js/dist/reveal.css\")", + "got_raw": "\"mkslides-assets/reveal-js/dist/reveal.css\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3853211009174312 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"mkslides-assets/reveal-js/dist/reveal.css\"", + "got_raw": "\"mkslides-assets/reveal-js/dist/reveal.css\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.3389830508474576 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"absolute_url_slideshow_theme_path\"", + "got_raw": "\"absolute_url_slideshow_theme_path\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.30357142857142855 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"simple\"", + "got_raw": "\"simple\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.11627906976744186 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"mkslides-assets/highlight-js-themes/vs.css\")", + "got_raw": "\"mkslides-assets/highlight-js-themes/vs.css\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.48 + }, + { + "repo": "MartenBE/mkslides", + "expected": "'')", + "got": "\"mkslides-assets/highlight-js-themes/vs.css\")", + "got_raw": "\"mkslides-assets/highlight-js-themes/vs.css\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "30)", + "got": "20)", + "got_raw": "20)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "any(\"SUCCESS\" in msg for msg in messages))", + "got": "len(messages)", + "got_raw": "len(messages)", + "exact_match": false, + "code_bleu": 0.18477554478349628, + "edit_similarity": 0.4 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "any(\"FAILURE\" in msg for msg in messages))", + "got": "any(\"ERROR\" in msg for msg in messages))", + "got_raw": "any(\"ERROR\" in msg for msg in messages))", + "exact_match": false, + "code_bleu": 0.7691605673134586, + "edit_similarity": 0.8780487804878049 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "any(\"Aborting\" in msg for msg in messages))", + "got": "len(messages)", + "got_raw": "len(messages)", + "exact_match": false, + "code_bleu": 0.18477554478349628, + "edit_similarity": 0.39285714285714285 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "any(\"FAILURE\" in msg and \"height\" in msg for msg in messages))", + "got": "messages)", + "got_raw": "messages)", + "exact_match": false, + "code_bleu": 0.09428090415820634, + "edit_similarity": 0.2535211267605634 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "any(\"FAILURE\" in msg and \"intersection\" in msg for msg in messages))", + "got": "messages)", + "got_raw": "messages)", + "exact_match": false, + "code_bleu": 0.09428090415820634, + "edit_similarity": 0.23376623376623376 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"refusal\"", + "got": "\"invalid_session\"", + "got_raw": "\"invalid_session\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"end_turn\"", + "got": "\"tool\"", + "got_raw": "\"tool\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.375 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "-1", + "got": "-1", + "got_raw": "-1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"running\"", + "got": "\"running\"", + "got_raw": "\"running\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "result.stdout", + "got": "result.output", + "got_raw": "result.output", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "result.stderr", + "got": "result.stderr", + "got_raw": "result.stderr", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "result.error.lower()", + "got": "result.error", + "got_raw": "result.error", + "exact_match": false, + "code_bleu": 0.36038417796587763, + "edit_similarity": 0.75 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "LLMProvider.OPENAI", + "got": "LLMProvider.OPENAI", + "got_raw": "LLMProvider.OPENAI", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "LLMProvider.ANTHROPIC", + "got": "LLMProvider.ANTHROPIC", + "got_raw": "LLMProvider.ANTHROPIC", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"get_weather\"", + "got": "\"get_weather\"", + "got_raw": "\"get_weather\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "3", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "skill.content", + "got": "skill.content", + "got_raw": "skill.content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "5.0", + "got": "5.0", + "got_raw": "5.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "15.0", + "got": "15.0", + "got_raw": "15.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "90.0", + "got": "90.0", + "got_raw": "90.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "10.0", + "got": "1.0", + "got_raw": "1.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "60.0", + "got": "10.0", + "got_raw": "10.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "30.0", + "got": "30.0", + "got_raw": "30.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "20.0", + "got": "20.0", + "got_raw": "20.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "result.content", + "got": "result.content", + "got_raw": "result.content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "6", + "got": "6", + "got_raw": "6", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "5", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"user\"", + "got": "\"user\"", + "got_raw": "\"user\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"system\"", + "got": "\"system\"", + "got_raw": "\"system\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "agent.messages", + "got": "agent.messages", + "got_raw": "agent.messages", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "result2.content", + "got": "result2.response", + "got_raw": "result2.response", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "len(agent.messages)", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.08333333333333333, + "edit_similarity": 0.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"MIT\"", + "got": "\"MIT\"", + "got_raw": "\"MIT\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"1.0\"", + "got": "\"1.0\"", + "got_raw": "\"1.0\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "prompt", + "got": "prompt", + "got_raw": "prompt", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"test-skill\"", + "got": "\"test-skill\"", + "got_raw": "\"test-skill\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "skill.content", + "got": "skill.content", + "got_raw": "skill.content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"Test Author\"", + "got": "\"Test Author\"", + "got_raw": "\"Test Author\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"get_skill\"", + "got": "\"get-skill\"", + "got_raw": "\"get-skill\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "result.content", + "got": "result.output", + "got_raw": "result.output", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7407407407407407 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "8", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "2", + "got": "", + "got_raw": "", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "4", + "got": "", + "got_raw": "", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "6", + "got": "characters", + "got_raw": " characters, 3 spaces", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "7", + "got": "", + "got_raw": " # 4 + 2", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "3", + "got": "4", + "got_raw": "4", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "9", + "got": "16", + "got_raw": "16", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "schema", + "got": "schema", + "got_raw": "schema", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"object\"", + "got": "\"object\"", + "got_raw": "\"object\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "[\"query\"]", + "got": "[\"query\"]", + "got_raw": "[\"query\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"function\"", + "got": "\"function\"", + "got_raw": "\"function\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "status_prop", + "got": "status_prop", + "got_raw": "status_prop", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "[\"location\"]", + "got": "[\"location\"]", + "got_raw": "[\"location\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"calculator\"", + "got": "\"calculate\"", + "got_raw": "\"calculate\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8695652173913043 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"get_weather\"", + "got": "\"MockWeatherTool\"", + "got_raw": "\"MockWeatherTool\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"Weather data\"", + "got": "\"Current weather in Tokyo: ...\"", + "got_raw": "\"Current weather in Tokyo: ...\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"search_database\"", + "got": "\"MockSearchTool\"", + "got_raw": "\"MockSearchTool\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.42424242424242425 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "result.content", + "got": "result.stdout", + "got_raw": "result.stdout", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"Test content\"", + "got": "\"Test content\"", + "got_raw": "\"Test content\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MiniMax-AI/Mini-Agent", + "expected": "\"Hello, Agent!\"", + "got": "\"Hello, Agent!\"", + "got_raw": "\"Hello, Agent!\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "0.1)", + "got": "0.0001)", + "got_raw": "0.0001)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "MrPowers/chispa", + "expected": "\"num2\")", + "got": "\"num2\")", + "got_raw": "\"num2\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "\"expected_name\")", + "got": "\"expected_name\")", + "got_raw": "\"expected_name\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "ColumnsNotEqualError)", + "got": "ColumnsNotEqualError)", + "got_raw": "ColumnsNotEqualError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "df2)", + "got": "df2)", + "got_raw": "df2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "0.1)", + "got": "ignore_columns=[\"expected_name\"])", + "got_raw": "ignore_columns=[\"expected_name\"])", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.05405405405405406 + }, + { + "repo": "MrPowers/chispa", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "ValueError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "MrPowers/chispa", + "expected": "df2.collect())", + "got": "df2.collect())", + "got_raw": "df2.collect())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "str(w[-1].message)", + "got": "str(w[-1].message)", + "got_raw": "str(w[-1].message)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "\"\\033[34mtest\\033[31m\"", + "got": "\"\\\\x1b[34mtest\\\\x1b[0m\"", + "got_raw": "\"\\\\x1b[34mtest\\\\x1b[0m\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6222222222222222 + }, + { + "repo": "MrPowers/chispa", + "expected": "DataFramesNotEqualError)", + "got": "DataFramesNotEqualError)", + "got_raw": "DataFramesNotEqualError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "\"\\033[97m\\033[4mtest\\033[0m\\033[31m\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "MrPowers/chispa", + "expected": "df2)", + "got": "df2", + "got_raw": "df2, ignore_row_order=True)", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "MrPowers/chispa", + "expected": "0.1)", + "got": "0.01)", + "got_raw": "0.01)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "MrPowers/chispa", + "expected": "expected_df)", + "got": "expected_df)", + "got_raw": "expected_df)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "df2.collect())", + "got": "df2.collect())", + "got_raw": "df2.collect())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "\"expected_name\")", + "got": "\"expected_name\")", + "got_raw": "\"expected_name\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "ColumnsNotEqualError)", + "got": "ColumnsNotEqualError)", + "got_raw": "ColumnsNotEqualError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "SchemasNotEqualError)", + "got": "SchemasNotEqualError)", + "got_raw": "SchemasNotEqualError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "df2.collect())", + "got": "df2.collect())", + "got_raw": "df2.collect())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "DataFramesNotEqualError)", + "got": "DataFramesNotEqualError)", + "got_raw": "DataFramesNotEqualError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "s2)", + "got": "s2)", + "got_raw": "s2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "Color.BLUE", + "got": "Color.BLUE", + "got_raw": "Color.BLUE", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "[Style.BOLD, Style.UNDERLINE]", + "got": "[\"bold\", \"underline\"]", + "got_raw": "[\"bold\", \"underline\"]", + "exact_match": false, + "code_bleu": 0.18037106415283274, + "edit_similarity": 0.24 + }, + { + "repo": "MrPowers/chispa", + "expected": "\"All elements in the list must be strings\"", + "got": "\"Format.from_list() expects a list of strings\"", + "got_raw": "\"Format.from_list() expects a list of strings\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5227272727272727 + }, + { + "repo": "MrPowers/chispa", + "expected": "\"Invalid style name: invalid_style. Valid style names are ['bold', 'underline', 'blink', 'invert', 'hide']\"", + "got": "\"Invalid style: invalid_style\"", + "got_raw": "\"Invalid style: invalid_style\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.43795620437956206 + }, + { + "repo": "MrPowers/chispa", + "expected": "\"Invalid color name: invalid_color. Valid color names are \" \"['black', 'red', 'green', 'yellow', 'blue', 'purple', 'cyan', 'light_gray', \" \"'dark_gray', 'light_red', 'light_green', 'light_yellow', 'light_blue', 'light_purple', \" \"'light_cyan', 'white']\"", + "got": "\"Invalid color: invalid_color\"", + "got_raw": "\"Invalid color: invalid_color\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.2049469964664311 + }, + { + "repo": "MrPowers/chispa", + "expected": "\"Invalid value: invalid_color. Valid values are colors: \" \"['black', 'red', 'green', 'yellow', 'blue', 'purple', 'cyan', 'light_gray', \" \"'dark_gray', 'light_red', 'light_green', 'light_yellow', 'light_blue', 'light_purple', \" \"'light_cyan', 'white'] and styles: ['bold', 'underline', 'blink', 'invert', 'hide']\"", + "got": "\"Invalid color: invalid_color\"", + "got_raw": "\"Invalid color: invalid_color\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.14035087719298245 + }, + { + "repo": "MrPowers/chispa", + "expected": "\"Invalid value: invalid_style. Valid values are colors: \" \"['black', 'red', 'green', 'yellow', 'blue', 'purple', 'cyan', 'light_gray', \" \"'dark_gray', 'light_red', 'light_green', 'light_yellow', 'light_blue', 'light_purple', \" \"'light_cyan', 'white'] and styles: ['bold', 'underline', 'blink', 'invert', 'hide']\"", + "got": "\"Invalid style 'invalid_style'\")", + "got_raw": "\"Invalid style 'invalid_style'\")", + "exact_match": false, + "code_bleu": 0.2259005009024612, + "edit_similarity": 0.05232558139534884 + }, + { + "repo": "MrPowers/chispa", + "expected": "None", + "got": "Style.NONE", + "got_raw": "Style.NONE", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "MrPowers/chispa", + "expected": "Color.RED", + "got": "Color.DEFAULT", + "got_raw": "Color.DEFAULT", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6363636363636364 + }, + { + "repo": "MrPowers/chispa", + "expected": "Color.BLUE", + "got": "Color.DEFAULT", + "got_raw": "Color.DEFAULT", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "MrPowers/chispa", + "expected": "Color.CYAN", + "got": "Color.CYAN", + "got_raw": "Color.CYAN", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "Color.GREEN", + "got": "Color.GREEN", + "got_raw": "Color.GREEN", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "Color.YELLOW", + "got": "\"yellow\"", + "got_raw": "\"yellow\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.2 + }, + { + "repo": "MrPowers/chispa", + "expected": "Color.PURPLE", + "got": "\"purple\"", + "got_raw": "\"purple\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.1 + }, + { + "repo": "MrPowers/chispa", + "expected": "[Style.BLINK]", + "got": "[\"blink\"]", + "got_raw": "[\"blink\"]", + "exact_match": false, + "code_bleu": 0.29999999999999993, + "edit_similarity": 0.2727272727272727 + }, + { + "repo": "MrPowers/chispa", + "expected": "[Style.UNDERLINE]", + "got": "Style.NONE", + "got_raw": "Style.NONE", + "exact_match": false, + "code_bleu": 0.3567621345008163, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "MrPowers/chispa", + "expected": "[Style.INVERT, Style.HIDE]", + "got": "[Style.INVERT, Style.HIDE]", + "got_raw": "[Style.INVERT, Style.HIDE]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MrPowers/chispa", + "expected": "[Style.BOLD, Style.UNDERLINE]", + "got": "Style.BOLD_UNDERLINE", + "got_raw": "Style.BOLD_UNDERLINE", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.7755102040816326 + }, + { + "repo": "MrPowers/chispa", + "expected": "\"Invalid style name: invalid_style. Valid style names are ['bold', 'underline', 'blink', 'invert', 'hide']\"", + "got": "\"Invalid style 'invalid_style'\")", + "got_raw": "\"Invalid style 'invalid_style'\")", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.4316546762589928 + }, + { + "repo": "MrPowers/chispa", + "expected": "\"Invalid color name: invalid_color. Valid color names are \" \"['black', 'red', 'green', 'yellow', 'blue', 'purple', 'cyan', 'light_gray', \" \"'dark_gray', 'light_red', 'light_green', 'light_yellow', 'light_blue', 'light_purple', \" \"'light_cyan', 'white']\"", + "got": "\"Invalid color: invalid_color\"", + "got_raw": "\"Invalid color: invalid_color\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.2049469964664311 + }, + { + "repo": "MrPowers/chispa", + "expected": "expected_string", + "got": "expected_string", + "got_raw": "expected_string", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "beta)", + "got": "beta)", + "got_raw": "beta)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "alpha)", + "got": "alpha)", + "got_raw": "alpha)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "greedy)", + "got": "greedy)", + "got_raw": "greedy)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "better_history)", + "got": "better_history)", + "got_raw": "better_history)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "lookback_period)", + "got": "lookback_period)", + "got_raw": "lookback_period)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "(ValueError, TypeError))", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.6285714285714286 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "eta)", + "got": "eta)", + "got_raw": "eta)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "gamma)", + "got": "gamma)", + "got_raw": "gamma)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "[7100])", + "got": "[1.0])", + "got_raw": "[1.0])", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "[7100, 7200])", + "got": "[7100, 7200])", + "got_raw": "[7100, 7200])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "[7100, 7200, 7120])", + "got": "[7100, 7200, 7120])", + "got_raw": "[7100, 7200, 7120])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "[7100, 7200, 7120, 7100])", + "got": "[7100, 7200, 7120, 7100])", + "got_raw": "[7100, 7200, 7120, 7100])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "[7200, 7120, 7100, 7200])", + "got": "[7100, 7200, 7120, 7100, 7200])", + "got_raw": "[7100, 7200, 7120, 7100, 7200])", + "exact_match": false, + "code_bleu": 0.7539221180326288, + "edit_similarity": 0.8928571428571429 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "beta)", + "got": "beta)", + "got_raw": "beta)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "alpha)", + "got": "alpha)", + "got_raw": "alpha)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "gamma)", + "got": "gamma)", + "got_raw": "gamma)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "delta)", + "got": "delta)", + "got_raw": "delta)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "end)", + "got": "end)", + "got_raw": "end)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "step)", + "got": "step)", + "got_raw": "step)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "start)", + "got": "start)", + "got_raw": "start)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "method)", + "got": "method)", + "got_raw": "method)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "ValueError)", + "got": "AssertionError)", + "got_raw": "AssertionError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5384615384615384 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "end)", + "got": "end)", + "got_raw": "end)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "step)", + "got": "step)", + "got_raw": "step)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "start)", + "got": "start)", + "got_raw": "start)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "method)", + "got": "method)", + "got_raw": "method)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "exp_end)", + "got": "exp_end)", + "got_raw": "exp_end)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "exp_step)", + "got": "exp_step)", + "got_raw": "exp_step)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "exp_start)", + "got": "exp_start)", + "got_raw": "exp_start)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "ValueError)", + "got": "AssertionError)", + "got_raw": "AssertionError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5384615384615384 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "end)", + "got": "end)", + "got_raw": "end)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "step)", + "got": "step)", + "got_raw": "step)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "start)", + "got": "start)", + "got_raw": "start)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "sa_end)", + "got": "sa_end)", + "got_raw": "sa_end)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "sa_step)", + "got": "sa_step)", + "got_raw": "sa_step)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "sa_start)", + "got": "sa_start)", + "got_raw": "sa_start)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "\"linear\")", + "got": "\"linear\")", + "got_raw": "\"linear\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "\"exponential\")", + "got": "\"autofit\")", + "got_raw": "\"autofit\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "ValueError)", + "got": "AttributeError)", + "got_raw": "AttributeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "update(1, 0.5, uc))", + "got": "update(1, 0.5, uc))", + "got_raw": "update(1, 0.5, uc))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "alpha)", + "got": "alpha)", + "got_raw": "alpha)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "scores)", + "got": "scores)", + "got_raw": "scores)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "(0, 0))", + "got": "(0, 0))", + "got_raw": "(0, 0))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "(1, 0))", + "got": "(1, 0))", + "got_raw": "(1, 0))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "(0, 0))", + "got": "(1, 0))", + "got_raw": "(1, 0))", + "exact_match": false, + "code_bleu": 0.6434588841607617, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "(1, 0))", + "got": "(0, 1))", + "got_raw": "(0, 1))", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "ValueError)", + "got": "AssertionError)", + "got_raw": "AssertionError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5384615384615384 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "[5, 0, 3, 0])", + "got": "[0.0, 0.0, 0.0, 0.0])", + "got_raw": "[0.0, 0.0, 0.0, 0.0])", + "exact_match": false, + "code_bleu": 0.20504572236241866, + "edit_similarity": 0.5294117647058824 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "AttributeError)", + "got": "AttributeError)", + "got_raw": "AttributeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "0.25)", + "got": "np.array([[0.25", + "got_raw": "np.array([[0.25, 0.25], [0.25, 0.25]])", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.4 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "1 / 3)", + "got": "0.33333333)", + "got_raw": "0.33333333)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "decay)", + "got": "decay)", + "got_raw": "decay)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "scores)", + "got": "scores)", + "got_raw": "scores)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "(1, 2))", + "got": "(2, 2))", + "got_raw": "(2, 2))", + "exact_match": false, + "code_bleu": 0.6434588841607617, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "num_repair)", + "got": "num_repair)", + "got_raw": "num_repair)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "num_destroy)", + "got": "num_destroy)", + "got_raw": "num_destroy)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "op_coupling)", + "got": "op_coupling)", + "got_raw": "op_coupling)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "expected[0])", + "got": "expected)", + "got_raw": "expected)", + "exact_match": false, + "code_bleu": 0.23784142300054423, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "expected[1])", + "got": "expected[1])", + "got_raw": "expected[1])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "seg_length)", + "got": "seg_length)", + "got_raw": "seg_length)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "expected[0])", + "got": "expected)", + "got_raw": "expected)", + "exact_match": false, + "code_bleu": 0.23784142300054423, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "expected[1])", + "got": "expected[1])", + "got_raw": "expected[1])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "max_iterations)", + "got": "max_iterations)", + "got_raw": "max_iterations)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "max_runtime)", + "got": "max_runtime)", + "got_raw": "max_runtime)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "max_iterations)", + "got": "max_iterations)", + "got_raw": "max_iterations)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "0)", + "got": "1.0)", + "got_raw": "1.0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "count)", + "got": "count)", + "got_raw": "count)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "desired)", + "got": "desired)", + "got_raw": "desired)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "ValueError)", + "got": "AssertionError)", + "got_raw": "AssertionError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5384615384615384 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "max_runtime)", + "got": "max_runtime)", + "got_raw": "max_runtime)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "max_iterations)", + "got": "max_iterations", + "got_raw": "max_iterations + 1)", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.9655172413793104 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "\"repair_operator\")", + "got": "\"repair_operator\")", + "got_raw": "\"repair_operator\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "\"destroy_operator\")", + "got": "\"destroy_operator\")", + "got_raw": "\"destroy_operator\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "max_iterations + 1)", + "got": "max_iterations + 1)", + "got_raw": "max_iterations + 1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "1)", + "got": "[time", + "got_raw": "[time for time in range(1, 101)])", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "99)", + "got": "100)", + "got_raw": "100)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "count)", + "got": "count", + "got_raw": "count, statistics.get_destroy_operator_count(\"destroy_test\", idx))", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "N-Wouda/ALNS", + "expected": "objective)", + "got": "objective)", + "got_raw": "objective)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "3)", + "got": "2)", + "got_raw": "2)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "4)", + "got": "4)", + "got_raw": "4)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "0)", + "got": "\"unknown\")", + "got_raw": "\"unknown\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "1)", + "got": "3)", + "got_raw": "3)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "3)", + "got": "3)", + "got_raw": "3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "2)", + "got": "int(ResourceType.file))", + "got_raw": "int(ResourceType.file))", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.08 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "3)", + "got": "3)", + "got_raw": "3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "4)", + "got": "4)", + "got_raw": "4)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "0)", + "got": "0.0)", + "got_raw": "0.0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "3)", + "got": "\"zip\")", + "got_raw": "\"zip\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.25 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "5)", + "got": "5)", + "got_raw": "5)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "7)", + "got": "7)", + "got_raw": "7)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"\")", + "got": "\"\")", + "got_raw": "\"\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"\")", + "got": "\"/\")", + "got_raw": "\"/\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"\")", + "got": "\"\")", + "got_raw": "\"\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"w\")", + "got": "\"Mode('w')\")", + "got_raw": "\"Mode('w')\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"f\")", + "got": "b\"o\")", + "got_raw": "b\"o\")", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.8 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"gz\")", + "got": "\"gz\")", + "got_raw": "\"gz\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"bar\")", + "got": "\"bar\")", + "got_raw": "\"bar\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"baz\")", + "got": "\"baz\")", + "got_raw": "\"baz\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"foo\")", + "got": "\"foo\")", + "got_raw": "\"foo\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"baz\")", + "got": "\"baz\")", + "got_raw": "\"baz\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"foo\")", + "got": "\"foo\")", + "got_raw": "\"foo\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\".py\")", + "got": "\"py\")", + "got_raw": "\"py\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\".gz\")", + "got": "\"tar.gz\")", + "got_raw": "\"tar.gz\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"m3\")", + "got": "b\"m3\")", + "got_raw": "b\"m3\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"m2\")", + "got": "b\"m2\")", + "got_raw": "b\"m2\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"m1\")", + "got": "b\"m1\")", + "got_raw": "b\"m1\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"1.0\")", + "got": "\"1.0\")", + "got_raw": "\"1.0\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"baz\")", + "got": "\"baz\")", + "got_raw": "\"baz\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"bar\")", + "got": "\"bar\")", + "got_raw": "\"bar\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"baz\")", + "got": "\"foo/bar/baz\")", + "got_raw": "\"foo/bar/baz\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"foo\")", + "got": "\"foo\")", + "got_raw": "\"foo\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"bar\")", + "got": "\"foo/bar\")", + "got_raw": "\"foo/bar\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.75 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"baz\")", + "got": "\"baz\")", + "got_raw": "\"baz\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"baz\")", + "got": "b\"baz\")", + "got_raw": "b\"baz\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"foo\")", + "got": "\"foo\")", + "got_raw": "\"foo\")", + "exact_match": false, + "code_bleu": 0.4714045207910317, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"will\")", + "got": "\"will\")", + "got_raw": "\"will\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"devs\")", + "got": "\"devs\")", + "got_raw": "\"devs\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"foo\")", + "got": "b\"foo\")", + "got_raw": "b\"foo\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"foo/\")", + "got": "\"foo\")", + "got_raw": "\"foo\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"/bar\")", + "got": "\"foo\")", + "got_raw": "\"foo\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"hello\")", + "got": "b\"hello\")", + "got_raw": "b\"hello\")", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.9411764705882353 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"foofoo\")", + "got": "\"foofoo\")", + "got_raw": "\"foofoo\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"1 byte\")", + "got": "\"1 byte\")", + "got_raw": "\"1 byte\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"1.0 KB\")", + "got": "\"1.0 KB\")", + "got_raw": "\"1.0 KB\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"1.0 MB\")", + "got": "\"1.0 MB\")", + "got_raw": "\"1.0 MB\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"1.5 MB\")", + "got": "\"1.5 MB\")", + "got_raw": "\"1.5 MB\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"1.0 kB\")", + "got": "\"1.0 KB\")", + "got_raw": "\"1.0 KB\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"1.2 MB\")", + "got": "\"1.2 MB\")", + "got_raw": "\"1.2 MB\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"bar.py\")", + "got": "\"bar.py\")", + "got_raw": "\"bar.py\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"Hello\")", + "got": "b\"Hello\")", + "got_raw": "b\"Hello\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"foobar\")", + "got": "b\"foobar\")", + "got_raw": "b\"foobar\")", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"Source\")", + "got": "\"Source\")", + "got_raw": "\"Source\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "b\"foofoo\")", + "got": "b\"foofoo\")", + "got_raw": "b\"foofoo\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"oh dear\")", + "got": "\"FS Error: oh dear\")", + "got_raw": "\"FS Error: oh dear\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"0 bytes\")", + "got": "0)", + "got_raw": "0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "\"2 bytes\")", + "got": "\"2 bytes\")", + "got_raw": "\"2 bytes\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "p)", + "got": "p.names)", + "got_raw": "p.names)", + "exact_match": false, + "code_bleu": 0.3976353643835253, + "edit_similarity": 0.4 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "m2)", + "got": "m2)", + "got_raw": "m2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "pp)", + "got": "pp)", + "got_raw": "pp)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "fs)", + "got": "tarfile.open(fs._tar_file))", + "got_raw": "tarfile.open(fs._tar_file))", + "exact_match": false, + "code_bleu": 0.15619699684601276, + "edit_similarity": 0.2 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "bool)", + "got": "bool)", + "got_raw": "bool)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "list)", + "got": "set)", + "got_raw": "set)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "OSFS)", + "got": "OSFS)", + "got_raw": "OSFS)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "data1)", + "got": "data1)", + "got_raw": "data1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "data2)", + "got": "data2)", + "got_raw": "data2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "data3)", + "got": "data3)", + "got_raw": "data3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "data4)", + "got": "data4)", + "got_raw": "data4)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "FTPFS)", + "got": "FTPFS)", + "got_raw": "FTPFS)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "SubFS)", + "got": "FTPFS)", + "got_raw": "FTPFS)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "bytes)", + "got": "bytes)", + "got_raw": "bytes)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "NoURL)", + "got": "NoURL)", + "got_raw": "NoURL)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "NoURL)", + "got": "NoURL)", + "got_raw": "NoURL)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "copied)", + "got": "copied)", + "got_raw": "copied)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "levels)", + "got": "levels)", + "got_raw": "levels)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "parsed)", + "got": "parsed)", + "got_raw": "parsed)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "result)", + "got": "result)", + "got_raw": "result)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "IOError)", + "got": "IOError)", + "got_raw": "IOError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "cleanup)", + "got": "cleanup)", + "got_raw": "cleanup)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "FSError)", + "got": "FSError)", + "got_raw": "FSError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "OSError)", + "got": "OSError)", + "got_raw": "OSError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "FSError)", + "got": "FSError)", + "got_raw": "FSError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "MemoryFS)", + "got": "MemoryFS)", + "got_raw": "MemoryFS)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "MemoryFS)", + "got": "MemoryFS)", + "got_raw": "MemoryFS)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "contents)", + "got": "contents)", + "got_raw": "contents)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "text_type)", + "got": "text_type)", + "got_raw": "text_type)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "TypeError)", + "got": "errors.InvalidPathError)", + "got_raw": "errors.InvalidPathError)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.4117647058823529 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "exception)", + "got": "exception)", + "got_raw": "exception)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "0o777)", + "got": "0)", + "got_raw": "0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "0o755)", + "got": "0o755)", + "got_raw": "0o755)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "0o700)", + "got": "0o755)", + "got_raw": "0o755)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "0o764)", + "got": "0o754)", + "got_raw": "0o754)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "p.u_r)", + "got": "p.read)", + "got_raw": "p.read)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "args[0])", + "got": "exc)", + "got_raw": "exc)", + "exact_match": false, + "code_bleu": 0.2149139863647084, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "dates[0])", + "got": "dates[0])", + "got_raw": "dates[0])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "dates[1])", + "got": "dates[1])", + "got_raw": "dates[1])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "dates[2])", + "got": "dates[2])", + "got_raw": "dates[2])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "dates[3])", + "got": "dates[3])", + "got_raw": "dates[3])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "info.name)", + "got": "info.basic)", + "got_raw": "info.basic)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "mode.text)", + "got": "mode.readable())", + "got_raw": "mode.readable())", + "exact_match": false, + "code_bleu": 0.31239399369202553, + "edit_similarity": 0.5384615384615384 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "bar.is_dir)", + "got": "bar.is_dir)", + "got_raw": "bar.is_dir)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "top.is_dir)", + "got": "top.is_dir)", + "got_raw": "top.is_dir)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "os.SEEK_CUR)", + "got": "0)", + "got_raw": "0)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "os.SEEK_END)", + "got": "os.SEEK_END)", + "got_raw": "os.SEEK_END)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "os.SEEK_SET)", + "got": "os.SEEK_SET)", + "got_raw": "os.SEEK_SET)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "ftps_fs.tls)", + "got": "ftps_fs.ftp)", + "got_raw": "ftps_fs.ftp)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.mem_fs)", + "got": "self.mem_fs)", + "got_raw": "self.mem_fs)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "foo_info.raw)", + "got": "foo_info.namespaces)", + "got_raw": "foo_info.namespaces)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "bar_info.raw)", + "got": "bar_info.raw)", + "got_raw": "bar_info.raw)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "errors.NoURL)", + "got": "errors.NoURL)", + "got_raw": "errors.NoURL)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "six.text_type)", + "got": "six.text_type)", + "got_raw": "six.text_type)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "six.text_type)", + "got": "str)", + "got_raw": "str)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.lrucache)", + "got": "self.lrucache)", + "got_raw": "self.lrucache)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "six.text_type)", + "got": "six.text_type)", + "got_raw": "six.text_type)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.server.host)", + "got": "self.server.host)", + "got_raw": "self.server.host)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.features)", + "got": "self.server.responses)", + "got_raw": "self.server.responses)", + "exact_match": false, + "code_bleu": 0.3303164318013807, + "edit_similarity": 0.5641025641025641 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "io.BufferedWriter)", + "got": "io.TextIOWrapper)", + "got_raw": "io.TextIOWrapper)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.5142857142857142 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "io.BufferedReader)", + "got": "iotools.StreamReader)", + "got_raw": "iotools.StreamReader)", + "exact_match": false, + "code_bleu": 0.3976353643835253, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "tarfile.ReadError)", + "got": "tarfile.ReadError)", + "got_raw": "tarfile.ReadError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "errors.InvalidPath)", + "got": "errors.InvalidPath)", + "got_raw": "errors.InvalidPath)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[])", + "got": "sorted([\"foo.py\"", + "got_raw": "sorted([\"foo.py\", \"bar.py\", \"egg\", \"a\", \"b\", \"c\"]))", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.10526315789473684 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[])", + "got": "[", + "got_raw": "[ \"foo.txt\", \".\" ])", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[\"/\"])", + "got": "[", + "got_raw": "[ \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\",", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[\"./\"])", + "got": "[])", + "got_raw": "[])", + "exact_match": false, + "code_bleu": 0.47920732818470435, + "edit_similarity": 0.6 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[\"foo\"])", + "got": "[\"foo\"])", + "got_raw": "[\"foo\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[\"foo\"])", + "got": "[\"m1\"", + "got_raw": "[\"m1\", \"m2\"])", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[\"baz\"])", + "got": "[\"baz\"])", + "got_raw": "[\"baz\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "(None, None))", + "got": "(None, None))", + "got_raw": "(None, None))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "[\"spam.txt\"])", + "got": "[])", + "got_raw": "[])", + "exact_match": false, + "code_bleu": 0.47920732818470435, + "edit_similarity": 0.375 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "fspath(path)", + "got": "fspath(path)", + "got_raw": "fspath(path)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "f1.isatty())", + "got": "f1.isatty())", + "got_raw": "f1.isatty())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "m1.isclosed())", + "got": "mount_fs.closed)", + "got_raw": "mount_fs.closed)", + "exact_match": false, + "code_bleu": 0.26509024292235017, + "edit_similarity": 0.6 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "m2.isclosed())", + "got": "m2.isclosed())", + "got_raw": "m2.isclosed())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "m1.isclosed())", + "got": "m1.closed)", + "got_raw": "m1.closed)", + "exact_match": false, + "code_bleu": 0.3387551654364098, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "m2.isclosed())", + "got": "m2.isclosed())", + "got_raw": "m2.isclosed())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "matcher(\"foo.py\"))", + "got": "matcher(\"\"))", + "got_raw": "matcher(\"\"))", + "exact_match": false, + "code_bleu": 0.45180100180492244, + "edit_similarity": 0.8 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "matcher(\"foo.PY\"))", + "got": "matcher(\"bar.py\"))", + "got_raw": "matcher(\"bar.py\"))", + "exact_match": false, + "code_bleu": 0.45180100180492244, + "edit_similarity": 0.7222222222222222 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "matcher(\"FOO.py\"))", + "got": "matcher(\"foo.PY\"))", + "got_raw": "matcher(\"foo.PY\"))", + "exact_match": false, + "code_bleu": 0.45180100180492244, + "edit_similarity": 0.7222222222222222 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.exists(\"a\"))", + "got": "globber(\"*.py\"))", + "got_raw": "globber(\"*.py\"))", + "exact_match": false, + "code_bleu": 0.21449855283116095, + "edit_similarity": 0.3888888888888889 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self._contents(fs2))", + "got": "self._contents(fs2))", + "got_raw": "self._contents(fs2))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "check_readable(\"r\"))", + "got": "check_readable(\"r\"))", + "got_raw": "check_readable(\"r\"))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "check_readable(\"w\"))", + "got": "check_readable(\"w\"))", + "got_raw": "check_readable(\"w\"))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "check_readable(\"a\"))", + "got": "check_writable(\"r\"))", + "got_raw": "check_writable(\"r\"))", + "exact_match": false, + "code_bleu": 0.3860973950960897, + "edit_similarity": 0.8 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "check_writable(\"w\"))", + "got": "check_writable(\"w\"))", + "got_raw": "check_writable(\"w\"))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "check_writable(\"r\"))", + "got": "check_writable(\"rt\"))", + "got_raw": "check_writable(\"rt\"))", + "exact_match": false, + "code_bleu": 0.45180100180492244, + "edit_similarity": 0.975609756097561 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "src_fs.isempty(\"/\"))", + "got": "src_file1_info.modified", + "got_raw": "src_file1_info.modified", + "exact_match": false, + "code_bleu": 0.19362900077353815, + "edit_similarity": 0.37209302325581395 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "dst_fs.isdir(\"bar\"))", + "got": "dst_fs.exists(\"test.txt\"))", + "got_raw": "dst_fs.exists(\"test.txt\"))", + "exact_match": false, + "code_bleu": 0.32172944208038085, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "fs1.getsyspath(\"/\"))", + "got": "fs1)", + "got_raw": "fs1)", + "exact_match": false, + "code_bleu": 0.16988673071467442, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "fs2.getsyspath(\"/\"))", + "got": "fs2.getsyspath(\"/\"))", + "got_raw": "fs2.getsyspath(\"/\"))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "fs.isdir(\"foo/bar\"))", + "got": "fs.isdir(\"foo/bar\"))", + "got_raw": "fs.isdir(\"foo/bar\"))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "ftp_fs.isfile(\"foo\"))", + "got": "ftp_fs.isfile(join(directory", + "got_raw": "ftp_fs.isfile(join(directory, \"foo\")))", + "exact_match": false, + "code_bleu": 0.5169731539571706, + "edit_similarity": 0.6530612244897959 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "src_fs.exists(\"foo\"))", + "got": "src_file2_info.modified)", + "got_raw": "src_file2_info.modified)", + "exact_match": false, + "code_bleu": 0.2272202082191573, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "os.path.isdir(_path))", + "got": "os.path.isdir(_path))", + "got_raw": "os.path.isdir(_path))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.isdir(\"foo\"))", + "got": "self.fs.isdir(\"foo\"))", + "got_raw": "self.fs.isdir(\"foo\"))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.getinfo(\"/\"))", + "got": "self.fs.getinfo(\"/\"))", + "got_raw": "self.fs.getinfo(\"/\"))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.exists(\"egg\"))", + "got": "globber(\"*.py\"))", + "got_raw": "globber(\"*.py\"))", + "exact_match": false, + "code_bleu": 0.21449855283116095, + "edit_similarity": 0.3684210526315789 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "os.path.exists(_path))", + "got": "os.path.exists(_path))", + "got_raw": "os.path.exists(_path))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.isfile(\"foo\"))", + "got": "self.fs.isfile(\"foo/eggs.bin\"))", + "got_raw": "self.fs.isfile(\"foo/eggs.bin\"))", + "exact_match": false, + "code_bleu": 0.7071067811865475, + "edit_similarity": 0.8301886792452831 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.delegate_fs())", + "got": "self.fs.delegate_path(\"\"))", + "got_raw": "self.fs.delegate_path(\"\"))", + "exact_match": false, + "code_bleu": 0.4671379777282001, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "src.exists(\"file.txt\"))", + "got": "src.exists(\"file.txt\"))", + "got_raw": "src.exists(\"file.txt\"))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "dst.exists(\"file.txt\"))", + "got": "dst.exists(\"target.txt\"))", + "got_raw": "dst.exists(\"target.txt\"))", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "tmp.exists(\"file.txt\"))", + "got": "tmp.exists(\"file.txt\"))", + "got_raw": "tmp.exists(\"file.txt\"))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "output_file.getvalue())", + "got": "output_file.getvalue())", + "got_raw": "output_file.getvalue())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.fs.getinfo(\"foo\"))", + "got": "self.fs.getinfo(\"foo\"))", + "got_raw": "self.fs.getinfo(\"foo\"))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "re.match(translated, m))", + "got": "re.match(translated, \"/\" + m))", + "got_raw": "re.match(translated, \"/\" + m))", + "exact_match": false, + "code_bleu": 0.6340466277046861, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "fs.isdir(\"foo/bar/baz\"))", + "got": "fs.exists(\"foo/bar/baz/egg\"))", + "got_raw": "fs.exists(\"foo/bar/baz/egg\"))", + "exact_match": false, + "code_bleu": 0.32172944208038085, + "edit_similarity": 0.7924528301886793 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "os.path.isdir(_temp_dir))", + "got": "os.path.exists(_temp_dir))", + "got_raw": "os.path.exists(_temp_dir))", + "exact_match": false, + "code_bleu": 0.6580370064762462, + "edit_similarity": 0.8627450980392157 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.cached.isdir(\"foo\"))", + "got": "self.cached.isdir(\"foo\"))", + "got_raw": "self.cached.isdir(\"foo\"))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.cached.isdir(\"egg\"))", + "got": "self.cached.isdir(\"egg\"))", + "got_raw": "self.cached.isdir(\"egg\"))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.cached.isdir(\"spam\"))", + "got": "self.cached.isdir(\"egg/\"))", + "got_raw": "self.cached.isdir(\"egg/\"))", + "exact_match": false, + "code_bleu": 0.7071067811865475, + "edit_similarity": 0.8461538461538461 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "self.cached.isfile(\"egg\"))", + "got": "self.cached.isfile(\"egg\"))", + "got_raw": "self.cached.isfile(\"egg\"))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "None)", + "got": "-1.0)", + "got_raw": "-1.0)", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.2 + }, + { + "repo": "PyFilesystem/pyfilesystem2", + "expected": "None)", + "got": "None)", + "got_raw": "None)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "None", + "got": "False", + "got_raw": "False", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "0.0", + "got_raw": "0.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "3", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "3", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "1", + "got": "0.0", + "got_raw": "0.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "3", + "got": "0.0", + "got_raw": "0.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "2", + "got": "0.0", + "got_raw": "0.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "4", + "got": "0.0", + "got_raw": "0.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "5", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "3", + "got": "0.0", + "got_raw": "0.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "3", + "got": "0.0", + "got_raw": "0.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "7", + "got": "7", + "got_raw": "7", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "1", + "got": "[(\"00000000-0000-0000-0000-000000000001\"", + "got_raw": "[(\"00000000-0000-0000-0000-000000000001\", \"2023-01-01 09:00:00\")]", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.04878048780487805 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "5", + "got": "0.0", + "got_raw": "0.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "5", + "got": "0.0", + "got_raw": "0.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"\"", + "got": "\"recall(query='hello')\"", + "got_raw": "\"recall(query='hello')\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.16 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"\"", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"a\"", + "got": "\"b\"", + "got_raw": "\"b\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"42\"", + "got": "42", + "got_raw": "42", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"42\"", + "got": "\"42\"", + "got_raw": "\"42\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"52\"", + "got": "\"10", + "got_raw": "\"10 * 5 + 2\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"db\"", + "got": "\"schema\"", + "got_raw": "\"schema\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"jq\"", + "got": "\"jq\"", + "got_raw": "\"jq\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"ok\"", + "got": "\"ok\"", + "got_raw": "\"ok\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"hi\"", + "got": "\"hi\"", + "got_raw": "\"hi\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"ok\"", + "got": "\"ok\"", + "got_raw": "\"ok\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"str\"", + "got": "\"string\"", + "got_raw": "\"string\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"sig\"", + "got": "\"sig\"", + "got_raw": "\"sig\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"CEO\"", + "got": "\"CEO\"", + "got_raw": "\"CEO\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"CTO\"", + "got": "\"CTO\"", + "got_raw": "\"CTO\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"web\"", + "got": "\"web\"", + "got_raw": "\"web\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"abc\"", + "got": "\"abc\"", + "got_raw": "\"abc\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"xyz\"", + "got": "\"xyz\"", + "got_raw": "\"xyz\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"pdf\"", + "got": "\"pdf\"", + "got_raw": "\"pdf\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"rss\"", + "got": "\"RSS\"", + "got_raw": "\"RSS\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"web\"", + "got": "\"web\"", + "got_raw": "\"web\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"bar\"", + "got": "\"bar\"", + "got_raw": "\"bar\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"llm\"", + "got": "\"init\"", + "got_raw": "\"init\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"raw\"", + "got": "{\"mode\":", + "got_raw": "{\"mode\": \"raw\"}", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"web\"", + "got": "\"web\"", + "got_raw": "\"web\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"grok\"", + "got": "\"gsuite\"", + "got_raw": "\"gsuite\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.42857142857142855 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"high\"", + "got": "\"high\"", + "got_raw": "\"high\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"user\"", + "got": "\"User\"", + "got_raw": "\"User\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"done\"", + "got": "\"done\"", + "got_raw": "\"done\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"push\"", + "got": "\"push\"", + "got_raw": "\"push\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"test\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"test\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"tu_1\"", + "got": "\"tu_1\"", + "got_raw": "\"tu_1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "\"user\"", + "got": "\"user\"", + "got_raw": "\"user\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "7 + 64", + "got": "64", + "got_raw": "64", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "args.func", + "got": "args.func.__name__", + "got_raw": "args.func.__name__", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "result[0]", + "got": "result[0]", + "got_raw": "result[0]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "blocks[0]", + "got": "blocks[0]", + "got_raw": "blocks[0]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "env.locals", + "got": "env._history", + "got_raw": "env._history", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.45454545454545453 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "repl.locals", + "got": "repl.context", + "got_raw": "repl.context", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.5217391304347826 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "repl.locals", + "got": "repl.variables", + "got_raw": "repl.variables", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.64 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "row[\"error\"]", + "got": "row[\"error\"]", + "got_raw": "row[\"error\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "result.error", + "got": "result.error", + "got_raw": "result.error", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "result.stdout", + "got": "result", + "got_raw": "result", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.631578947368421 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "result.stderr", + "got": "result.stderr", + "got_raw": "result.stderr", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "row[\"content\"]", + "got": "row[\"content\"]", + "got_raw": "row[\"content\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "ToolCategory.WEB", + "got": "ToolCategory.WEB", + "got_raw": "ToolCategory.WEB", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "ToolCategory.WEB", + "got": "ToolCategory.WEB", + "got_raw": "ToolCategory.WEB", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "False", + "got_raw": "False", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "s", + "got": "s", + "got_raw": "s", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "p", + "got": "p", + "got_raw": "p", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "d", + "got": "d", + "got_raw": "d", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "h", + "got": "h", + "got_raw": "h", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "v2", + "got": "v2", + "got_raw": "v2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "c2", + "got": "c2", + "got_raw": "c2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "s2", + "got": "s2", + "got_raw": "s2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "k2", + "got": "k2", + "got_raw": "k2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "ts", + "got": "ts", + "got_raw": "ts", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "ns", + "got": "ns", + "got_raw": "ns", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "cfg", + "got": "cfg", + "got_raw": "cfg", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "att", + "got": "att", + "got_raw": "att", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "mid", + "got": "[{\"source_file\":", + "got_raw": "[{\"source_file\": src, \"chunk_index\": 0}]", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.10526315789473684 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "id3", + "got": "id3", + "got_raw": "id3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "cfg", + "got": "cfg", + "got_raw": "cfg", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "dsn", + "got": "dsn", + "got_raw": "dsn", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "dsn", + "got": "dsn", + "got_raw": "dsn", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "cfg", + "got": "cfg", + "got_raw": "cfg", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "row", + "got": "row", + "got_raw": "row", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "sql", + "got": "sql", + "got_raw": "sql", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "url", + "got": "url", + "got_raw": "url", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "url", + "got": "url", + "got_raw": "url", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "ids", + "got": "ids", + "got_raw": "ids", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "val", + "got": "val", + "got_raw": "val", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "ids", + "got": "ids", + "got_raw": "ids", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "ids", + "got": "ids", + "got_raw": "ids", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "str(wm_id)", + "got": "wm_id", + "got_raw": "wm_id", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "float(score)", + "got": "score", + "got_raw": "score", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.5882352941176471 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "float(trust)", + "got": "trust", + "got_raw": "trust", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.5882352941176471 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "len(verifier)", + "got": "len(verifier)", + "got_raw": "len(verifier)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "QuixiAI/Hexis", + "expected": "float(alignment)", + "got": "alignment", + "got_raw": "alignment", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.72 + }, + { + "repo": "agronholm/anyio", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "task_state.cancel_scope._tasks", + "got": "self._tasks", + "got_raw": "self._tasks", + "exact_match": false, + "code_bleu": 0.3567621345008163, + "edit_similarity": 0.4878048780487805 + }, + { + "repo": "agronholm/anyio", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "handle_exc", + "got": "handle_exc", + "got_raw": "handle_exc", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "RuntimeError, match=\"^foo$\")", + "got": "Exception))", + "got_raw": "Exception))", + "exact_match": false, + "code_bleu": 0.2259005009024612, + "edit_similarity": 0.20512820512820512 + }, + { + "repo": "agronholm/anyio", + "expected": "TypeError, match=\"returned self\")", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.46511627906976744 + }, + { + "repo": "agronholm/anyio", + "expected": "TypeError, match=\"returned a generator\")", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.4 + }, + { + "repo": "agronholm/anyio", + "expected": "TypeError, match=\"returned an async generator\")", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.3508771929824561 + }, + { + "repo": "agronholm/anyio", + "expected": "TypeError, match=\"did not return a context manager\")", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.3225806451612903 + }, + { + "repo": "agronholm/anyio", + "expected": "TypeError, match=\"returned a coroutine object instead of\")", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.29411764705882354 + }, + { + "repo": "agronholm/anyio", + "expected": "TypeError, match=\"did not return an async context manager\")", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.2898550724637681 + }, + { + "repo": "agronholm/anyio", + "expected": "RuntimeError, match=\"^this DummyContextManager has not been entered yet$\")", + "got": "cm as cm2:", + "got_raw": " cm as cm2:", + "exact_match": false, + "code_bleu": 0.2014250264904512, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "agronholm/anyio", + "expected": "RuntimeError, match=\"^this DummyContextManager has already been entered$\")", + "got": "cm as cm2:", + "got_raw": " cm as cm2:", + "exact_match": false, + "code_bleu": 0.2014250264904512, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "agronholm/anyio", + "expected": "RuntimeError, match=\"^this DummyAsyncContextManager has not been entered yet$\")", + "got": "cm as cm:", + "got_raw": " cm as cm:", + "exact_match": false, + "code_bleu": 0.2014250264904512, + "edit_similarity": 0.13636363636363635 + }, + { + "repo": "agronholm/anyio", + "expected": "RuntimeError, match=\"^this DummyAsyncContextManager has already been entered$\")", + "got": "pytest.raises(RuntimeError)", + "got_raw": " pytest.raises(RuntimeError)", + "exact_match": false, + "code_bleu": 0.24446151121745047, + "edit_similarity": 0.24528301886792453 + }, + { + "repo": "agronholm/anyio", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "agronholm/anyio", + "expected": "\"wait\"", + "got": "\"native_coro_part\"", + "got_raw": "\"native_coro_part\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4166666666666667 + }, + { + "repo": "agronholm/anyio", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "host_task.id", + "got": "host_task.id", + "got_raw": "host_task.id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "expected_name", + "got": "expected_name", + "got_raw": "expected_name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "\"tests.test_debugging.test_main_task_name..main\"", + "got": "\"main\"", + "got_raw": "\"main\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.1935483870967742 + }, + { + "repo": "agronholm/anyio", + "expected": "anyio.BrokenWorkerInterpreter", + "got": "anyio.BrokenWorkerInterpreter", + "got_raw": "anyio.BrokenWorkerInterpreter", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "0)", + "got": "500.102352)", + "got_raw": "500.102352)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "agronholm/anyio", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "math.inf)", + "got": "math.inf)", + "got_raw": "math.inf)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "deadline - fake_current_time)", + "got": "deadline - fake_current_time)", + "got_raw": "deadline - fake_current_time)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "uri", + "got": "uri", + "got_raw": "uri", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "100", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8 + }, + { + "repo": "agronholm/anyio", + "expected": "\"c:\"", + "got": "\"\\\\", + "got_raw": "\"\\\\", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "agronholm/anyio", + "expected": "path", + "got": "path", + "got_raw": "path", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "path2", + "got": "path2", + "got_raw": "path2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "path1", + "got": "path1", + "got_raw": "path1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "0o444", + "got": "0o444", + "got_raw": "0o444", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "0o600", + "got": "0o600", + "got_raw": "0o600", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "\"bar\"", + "got": "\"bar\"", + "got_raw": "\"bar\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "\".zip\"", + "got": "\".zip\"", + "got_raw": "\".zip\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "7", + "got": "7", + "got_raw": "7", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "6", + "got": "6", + "got_raw": "6", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "agronholm/anyio", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "\"a\"", + "got": "\"a\"", + "got_raw": "\"a\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "\"b\"", + "got": "\"b\"", + "got_raw": "\"b\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "index", + "got": "index", + "got_raw": "index", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "i + 1", + "got": "i", + "got_raw": "i", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "agronholm/anyio", + "expected": "cancel", + "got": "cancel", + "got_raw": "cancel", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "LookupError)", + "got": "RuntimeError)", + "got_raw": "RuntimeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.56 + }, + { + "repo": "agronholm/anyio", + "expected": "ValueError, match=\"This token has already been used\")", + "got": "RuntimeError)", + "got_raw": "RuntimeError)", + "exact_match": false, + "code_bleu": 0.17909498863725698, + "edit_similarity": 0.24242424242424243 + }, + { + "repo": "agronholm/anyio", + "expected": "ValueError, match=\"This token does not belong to this RunVar\")", + "got": "pytest.raises(RuntimeError)", + "got_raw": "pytest.raises(RuntimeError)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.20224719101123595 + }, + { + "repo": "agronholm/anyio", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "\"bogus error\"", + "got": "\"Exception during teardown: RuntimeError('failing fixture teardown')\"", + "got_raw": "\"Exception during teardown: RuntimeError('failing fixture teardown')\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.17073170731707318 + }, + { + "repo": "agronholm/anyio", + "expected": "result.stdout.str()", + "got": "\"::test_params\"", + "got_raw": "\"::test_params\"", + "exact_match": false, + "code_bleu": 0.07142857142857142, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "agronholm/anyio", + "expected": "signal.Signals.SIGUSR1", + "got": "signal.SIGUSR1", + "got_raw": "signal.SIGUSR1", + "exact_match": false, + "code_bleu": 0.4242640687119285, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "agronholm/anyio", + "expected": "signal.Signals.SIGUSR2", + "got": "signal.Signals.SIGUSR2", + "got_raw": "signal.Signals.SIGUSR2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "agronholm/anyio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "[]", + "got": "gai_result", + "got_raw": "gai_result", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "agronholm/anyio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"cba\"", + "got": "\"cba\"", + "got_raw": "\"cba\"", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "agronholm/anyio", + "expected": "str(sid)", + "got": "str(sid)", + "got_raw": "str(sid)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"ready\"", + "got": "\"ready\"", + "got_raw": "\"ready\"", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "agronholm/anyio", + "expected": "env[\"foo\"]", + "got": "\"bar\"", + "got_raw": "\"bar\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "agronholm/anyio", + "expected": "EndOfStream)", + "got": "ClosedResourceError)", + "got_raw": "ClosedResourceError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.125 + }, + { + "repo": "agronholm/anyio", + "expected": "2", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "agronholm/anyio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "agronholm/anyio", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "0", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "agronholm/anyio", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "data", + "got": "data", + "got_raw": "data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "content", + "got": "content", + "got_raw": "content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "ValueError)", + "got": "IOError)", + "got_raw": "IOError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.631578947368421 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"helloworld\"", + "got": "b\"hello\"", + "got_raw": "b\"hello\"", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7619047619047619 + }, + { + "repo": "agronholm/anyio", + "expected": "FileNotFoundError)", + "got": "RuntimeError)", + "got_raw": "RuntimeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5161290322580645 + }, + { + "repo": "agronholm/anyio", + "expected": "tempfile.gettempdir()", + "got": "os.path.join(os.path.sep", + "got_raw": "os.path.join(os.path.sep, \"tmp\")", + "exact_match": false, + "code_bleu": 0.12605968092174913, + "edit_similarity": 0.17777777777777778 + }, + { + "repo": "agronholm/anyio", + "expected": "tempfile.gettempdirb()", + "got": "tempfile.gettempdir()", + "got_raw": "tempfile.gettempdir()", + "exact_match": false, + "code_bleu": 0.45180100180492244, + "edit_similarity": 0.9767441860465116 + }, + { + "repo": "agronholm/anyio", + "expected": "[\"b\", \"a\"]", + "got": "[\"b\", \"a\"]", + "got_raw": "[\"b\", \"a\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "interpreter_id_2", + "got": "interpreter_id_2", + "got_raw": "interpreter_id_2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "main_interpreter_id", + "got": "main_interpreter_id", + "got_raw": "main_interpreter_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "ValueError, match=\"invalid literal for int\")", + "got": "RuntimeError)", + "got_raw": "RuntimeError)", + "exact_match": false, + "code_bleu": 0.17909498863725698, + "edit_similarity": 0.2807017543859649 + }, + { + "repo": "agronholm/anyio", + "expected": "sys.path", + "got": "sys.path", + "got_raw": "sys.path", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "worker_pid", + "got": "worker_pid", + "got_raw": "worker_pid", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "os.getpid()", + "got": "os.getpid()", + "got_raw": "os.getpid()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "worker_pid2", + "got": "worker_pid2", + "got_raw": "worker_pid2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "real_worker", + "got": "fake_idle_process", + "got_raw": "fake_idle_process", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "agronholm/anyio", + "expected": "LookupError)", + "got": "RuntimeError)", + "got_raw": "RuntimeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.56 + }, + { + "repo": "agronholm/anyio", + "expected": "ValueError, match=\"invalid literal for int\")", + "got": "RuntimeError)", + "got_raw": "RuntimeError)", + "exact_match": false, + "code_bleu": 0.17909498863725698, + "edit_similarity": 0.2807017543859649 + }, + { + "repo": "agronholm/anyio", + "expected": "[ \"b\", \"a\", ]", + "got": "[\"b\", \"a\"]", + "got_raw": "[\"b\", \"a\"]", + "exact_match": false, + "code_bleu": 0.6627256073058756, + "edit_similarity": 0.8695652173913043 + }, + { + "repo": "agronholm/anyio", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "agronholm/anyio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "6", + "got": "6", + "got_raw": "6", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "None", + "got": "asyncio", + "got_raw": "asyncio", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "agronholm/anyio", + "expected": "portal2", + "got": "portal2", + "got_raw": "portal2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "no_other_refs()", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.30120066786994826, + "edit_similarity": 0.0 + }, + { + "repo": "agronholm/anyio", + "expected": "anyio_backend_name", + "got": "\"anyio\"", + "got_raw": "\"anyio\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "agronholm/anyio", + "expected": "expected_last_active", + "got": "expected_last_active", + "got_raw": "expected_last_active", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "KeyError, match=\"^'foo'$\")", + "got": "KeyError)", + "got_raw": "KeyError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5142857142857142 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"fg\"", + "got": "b\"def\"", + "got_raw": "b\"def\"", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"ab\"", + "got": "b\"ab\"", + "got_raw": "b\"ab\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"cd\"", + "got": "b\"cd\"", + "got_raw": "b\"cd\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"abc\"", + "got": "b\"abcd\"", + "got_raw": "b\"abcd\"", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"abcd\"", + "got": "b\"abcd\"", + "got_raw": "b\"abcd\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"abcdefgh\"", + "got": "b\"abcd\"", + "got_raw": "b\"abcd\"", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "agronholm/anyio", + "expected": "EndOfStream)", + "got": "EndOfStream)", + "got_raw": "EndOfStream)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"xxxfooabcd\"", + "got": "b\"abcd\"", + "got_raw": "b\"abcd\"", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7 + }, + { + "repo": "agronholm/anyio", + "expected": "IncompleteRead)", + "got": "IncompleteRead)", + "got_raw": "IncompleteRead)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "ClosedResourceError)", + "got": "ClosedResourceError)", + "got_raw": "ClosedResourceError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "2", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "agronholm/anyio", + "expected": "5", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"lo\"", + "got": "b\"lo\"", + "got_raw": "b\"lo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"Hel\"", + "got": "b\"Hel\"", + "got_raw": "b\"Hel\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "fileno", + "got": "fileno", + "got_raw": "fileno", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"llo\"", + "got": "b\"lo\"", + "got_raw": "b\"lo\"", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "agronholm/anyio", + "expected": "\"Hello\"", + "got": "\"Hello\"", + "got_raw": "\"Hello\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "file_path", + "got": "file_path", + "got_raw": "file_path", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "EndOfStream)", + "got": "ClosedResourceError)", + "got_raw": "ClosedResourceError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.125 + }, + { + "repo": "agronholm/anyio", + "expected": "\"Hello, World!\"", + "got": "\"Hello, World!\"", + "got_raw": "\"Hello, World!\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "ClosedResourceError)", + "got": "ClosedResourceError)", + "got_raw": "ClosedResourceError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "agronholm/anyio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "\"item\"", + "got": "\"item\"", + "got_raw": "\"item\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "\"test\"", + "got": "\"hello\"", + "got_raw": "\"hello\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "agronholm/anyio", + "expected": "\"anyio\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "agronholm/anyio", + "expected": "\"hello\"", + "got": "\"hello\"", + "got_raw": "\"hello\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "\"world\"", + "got": "\"world\"", + "got_raw": "\"world\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"\"", + "got": "b\"\"", + "got_raw": "b\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"hel\"", + "got": "b\"hel\"", + "got_raw": "b\"hel\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "\"hello\"", + "got": "\"hello\"", + "got_raw": "\"hello\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "\"world\"", + "got": "\"world\"", + "got_raw": "\"world\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "send_stream", + "got": "send_stream", + "got_raw": "send_stream", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"hello, world\"", + "got": "b\"hello\"", + "got_raw": "b\"hello\"", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6956521739130435 + }, + { + "repo": "agronholm/anyio", + "expected": "ClosedResourceError)", + "got": "ClosedResourceError)", + "got_raw": "ClosedResourceError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "\"\u00f6\"", + "got": "\"\u00f6\"", + "got_raw": "\"\u00f6\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "\"\u00e5\u00e4\"", + "got": "\"\u00f6\"", + "got_raw": "\"\u00f6\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"?\"", + "got": "b\"\\xc2\\x82\"", + "got_raw": "b\"\\xc2\\x82\"", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4 + }, + { + "repo": "agronholm/anyio", + "expected": "\"\u00e6\u00f8\"", + "got": "\"\u00e5\u00f6\"", + "got_raw": "\"\u00e5\u00f6\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "agronholm/anyio", + "expected": "\"\ufffd\ufffd\ufffd\"", + "got": "\"\u00e5\u00e4\u00f6\"", + "got_raw": "\"\u00e5\u00e4\u00f6\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "agronholm/anyio", + "expected": "\"hello\"", + "got": "\"hello\"", + "got_raw": "\"hello\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"\\xc3\\xa5\\xc3\\xa4\\xc3\\xb6\"", + "got": "b\"\u00e5\u00e4\u00f6\"", + "got_raw": "b\"\u00e5\u00e4\u00f6\"", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "agronholm/anyio", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "exc", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "agronholm/anyio", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "\"h2\"", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "agronholm/anyio", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"olleh\"", + "got": "b\"olleh\"", + "got_raw": "b\"olleh\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "\"dummy2\"", + "got": "\"dummy2\"", + "got_raw": "\"dummy2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/anyio", + "expected": "b\"hello\"", + "got": "b\"hello\"", + "got_raw": "b\"hello\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "expected_count)", + "got": "expected_count)", + "got_raw": "expected_count)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "*expected_violations_when_fixing)", + "got": "*expected_violations_when_fixing)", + "got_raw": "*expected_violations_when_fixing)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "*expected_violations_when_checking)", + "got": "expected_violations_when_checking)", + "got_raw": "expected_violations_when_checking)", + "exact_match": false, + "code_bleu": 0.4714045207910317, + "edit_similarity": 0.9855072463768116 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "ALL_TAGS", + "got": "ALL_TAGS", + "got_raw": "ALL_TAGS", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "*name_contents)", + "got": "style.path_from_resources_root)", + "got_raw": "style.path_from_resources_root)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.08695652173913043 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "expected_pylintrc)", + "got": "=", + "got_raw": " = project_default", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "datadir / \"real.yaml\")", + "got": "uss(\"real.toml\")", + "got_raw": "uss(\"real.toml\")", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.5263157894736842 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "0)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "1)", + "got": "3)", + "got_raw": "3)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "3)", + "got": "0)", + "got_raw": "0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "4)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "5)", + "got": "0)", + "got_raw": "0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "(expected_enum, expected_timedelta)", + "got": "expected_enum, expected_timedelta", + "got_raw": "expected_enum, expected_timedelta", + "exact_match": false, + "code_bleu": 0.5045378491522287, + "edit_similarity": 0.9705882352941176 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "unchanged)", + "got": "=", + "got_raw": " = ProjectMock(tmp_path)", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "f\"\"\" [{CONFIG_TOOL_NITPICK_KEY}] style = [ \"my-style.toml\",] \"\"\")", + "got": "_config_path = project.config_path", + "got_raw": "_config_path = project.config_path", + "exact_match": false, + "code_bleu": 0.12409640285041212, + "edit_similarity": 0.1414141414141414 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "f\"\"\" [{CONFIG_TOOL_NITPICK_KEY}]", + "got": "_config_path = tmp_path / DOT_NITPICK_TOML", + "got_raw": "_config_path = tmp_path / DOT_NITPICK_TOML", + "exact_match": false, + "code_bleu": 0.14348430291633957, + "edit_similarity": 0.2972972972972973 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "f\"\"\" {before}[{CONFIG_TOOL_NITPICK_KEY}]{after}", + "got": "_config_file = project.config_file(config_file)", + "got_raw": "_config_file = project.config_file(config_file)", + "exact_match": false, + "code_bleu": 0.07705778205430676, + "edit_similarity": 0.0851063829787234 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "f\"\"\" [{CONFIG_TOOL_NITPICK_KEY}]", + "got": "_config = project.read_config()", + "got_raw": "_config = project.read_config()", + "exact_match": false, + "code_bleu": 0.1366723722402612, + "edit_similarity": 0.06349206349206349 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "path", + "got": "path", + "got_raw": "path", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "set()", + "got": "set()", + "got_raw": "set()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "message.strip().casefold()", + "got": "message", + "got_raw": "message", + "exact_match": false, + "code_bleu": 0.0660670619445956, + "edit_similarity": 0.42424242424242425 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "{ some_directory / \"README.md\", }", + "got": "{ some_directory / \"README.md\" }", + "got_raw": "{ some_directory / \"README.md\" }", + "exact_match": false, + "code_bleu": 0.6627256073058756, + "edit_similarity": 0.9846153846153847 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "{ some_directory / \"src\" / \"module.py\", }", + "got": "{", + "got_raw": "{", + "exact_match": false, + "code_bleu": 0.07432544468767006, + "edit_similarity": 0.047619047619047616 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "{ some_directory / \"file1.txt\", some_directory / \"src\" / \"module.py\", }", + "got": "{ \"file1.txt\", \"README.md\", \"src/module.py\", }", + "got_raw": "{ \"file1.txt\", \"README.md\", \"src/module.py\", }", + "exact_match": false, + "code_bleu": 0.18517460226745275, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "{ some_directory / GIT_IGNORE, some_directory / \"README.md\", some_directory / \"src\" / \"module.py\", }", + "got": "{ \"file1.txt\", \"README.md\", \"src/module.py\", }", + "got_raw": "{ \"file1.txt\", \"README.md\", \"src/module.py\", }", + "exact_match": false, + "code_bleu": 0.14433756729740643, + "edit_similarity": 0.5205479452054794 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "{ some_directory / \"file1.txt\", some_directory / \"README.md\", some_directory / \"src\" / \"module.py\", }", + "got": "set()", + "got_raw": "set()", + "exact_match": false, + "code_bleu": 0.0712345955298368, + "edit_similarity": 0.05660377358490566 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "original_file)", + "got": "original_file)", + "got_raw": "original_file)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "expected_generic_ini)", + "got": "generic.ini\"", + "got_raw": "generic.ini\", expected_generic_ini)", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.6060606060606061 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "expected_editor_config)", + "got": "expected_setup_cfg)", + "got_raw": "expected_setup_cfg)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7619047619047619 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "datadir / \"3-expected-setup.cfg\")", + "got": "with_file(datadir / \"multiline_comment.ini\")", + "got_raw": "with_file(datadir / \"multiline_comment.ini\")", + "exact_match": false, + "code_bleu": 0.31239399369202553, + "edit_similarity": 0.38961038961038963 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "datadir / \"2-expected-editorconfig.ini\")", + "got": "", + "got_raw": "", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"\"\" [flake8] max-line-length = 100 max-complexity = 10 \"\"\")", + "got": "= ProjectMock(tmp_path)", + "got_raw": " = ProjectMock(tmp_path)", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.17073170731707318 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"\"\" [food] eat = spam,eggs,cheese,ham,salt drink = wine , bier , water \"\"\")", + "got": "= ProjectMock(tmp_path)", + "got_raw": " = ProjectMock(tmp_path)", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.12244897959183673 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"\"\" [mypy] ignore_missing_imports = true [flake8] max-line-length = 120 [isort] line_length = 120 \"\"\")", + "got": "= ProjectMock(tmp_path)", + "got_raw": " = ProjectMock(tmp_path)", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.16 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "Fuss( True, PYTHON_SETUP_CFG, 324, \": section [flake8] has some missing key/value pairs. Use this:\", \"\"\" [flake8] new = value \"\"\", ))", + "got": "= ProjectMock(tmp_path)", + "got_raw": " = ProjectMock(tmp_path)", + "exact_match": false, + "code_bleu": 0.094417981167399, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "Fuss( False, PYTHON_SETUP_CFG, Violations.PARSING_ERROR.code, \": parsing error (ParsingError): Source contains parsing errors: 'simulating a captured error'\", ))", + "got": "= ProjectMock()", + "got_raw": " = ProjectMock()", + "exact_match": false, + "code_bleu": 0.09940884109588133, + "edit_similarity": 0.11363636363636363 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"\"\" [mypy]", + "got": "= ProjectMock(tmp_path)", + "got_raw": " = ProjectMock(tmp_path)", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "Fuss( False, PYTHON_SETUP_CFG, Violations.PARSING_ERROR.code, f\": parsing error (DuplicateOptionError): While reading from {project.path_for(PYTHON_SETUP_CFG)!r} \" f\"[line 3]: option 'easy' in section 'abc' already exists\", ))", + "got": "_file = project.add_file(\"test.ini\")", + "got_raw": "_file = project.add_file(\"test.ini\")", + "exact_match": false, + "code_bleu": 0.08029822350243528, + "edit_similarity": 0.07633587786259542 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "Fuss( False, PYTHON_SETUP_CFG, 321, \" was not found. Create it with this content:\", \"\"\" [flake8] max-line-length = 85 max-complexity = 12 ignore = D100,D101,D102,D103,D104,D105,D106,D107,D202,E203,W503 select = E241,C,E,F,W,B,B9 \"\"\", ))", + "got": "= ProjectMock(tmp_path)", + "got_raw": " = ProjectMock(tmp_path)", + "exact_match": false, + "code_bleu": 0.094417981167399, + "edit_similarity": 0.06177606177606178 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "expected_package_json)", + "got": "(\"package.json\")", + "got_raw": "(\"package.json\")", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.631578947368421 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "datadir / \"3-expected.json\")", + "got": "_path)", + "got_raw": "_path)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.17647058823529413 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "err", + "got": "\"Using", + "got_raw": "\"Using the default style\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "retv", + "got": "retv", + "got_raw": "retv", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"something\"", + "got": "\"something\"", + "got_raw": "\"something\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "SystemExit)", + "got": "SystemExit)", + "got_raw": "SystemExit)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"NITPICK_SOME_OPTION\"", + "got": "\"nitpick-some-option\"", + "got_raw": "\"nitpick-some-option\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"--nitpick-multi-word\"", + "got": "\"--multi-word\"", + "got_raw": "\"--multi-word\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "app2", + "got": "app2", + "got_raw": "app2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "root", + "got": "root", + "got_raw": "root", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "error)", + "got": "error)", + "got_raw": "error)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "str(err)", + "got": "err.value.args[0]", + "got_raw": "err.value.args[0]", + "exact_match": false, + "code_bleu": 0.16036590969929357, + "edit_similarity": 0.24 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "apps_dir", + "got": "apps_dir", + "got_raw": "apps_dir", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "TypeError)", + "got": "QuitComplainingError)", + "got_raw": "QuitComplainingError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.45161290322580644 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "caplog.messages", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "QuitComplainingError)", + "got": "QuitComplainingError)", + "got_raw": "QuitComplainingError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "apps_dir / PYTHON_MANAGE_PY", + "got": "tmp_path", + "got_raw": "tmp_path", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.11428571428571428 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "f\"NIP102 No Python file was found on the root dir and subdir of {str(project.root_dir)!r}\")", + "got": "= Nitpick(project)", + "got_raw": " = Nitpick(project)", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.25688073394495414 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "2)", + "got": "=", + "got_raw": " = ProjectMock(tmp_path)", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "1)", + "got": "file", + "got_raw": "file = project.project_file", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "None)", + "got": "_style", + "got_raw": "_style = project.style", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"nitpick\"", + "got": "\"nitpick\"", + "got_raw": "\"nitpick\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "import_path", + "got": "import_path", + "got_raw": "import_path", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"andreoliwa\"", + "got": "\"andreoliwa\"", + "got_raw": "\"andreoliwa\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "git_reference", + "got": "git_reference", + "got_raw": "git_reference", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "resource_name", + "got": "resource_name", + "got_raw": "resource_name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "RuntimeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "furl(expected_url)", + "got": "expected_url", + "got_raw": "expected_url", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.8 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "f\"token {file_token}\"", + "got": "f\"token {file_token}\"", + "got_raw": "f\"token {file_token}\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "expected_content_path", + "got": "expected_content_path", + "got_raw": "expected_content_path", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "mocked_version == \"0.5.3\")", + "got": "mocked_version)", + "got_raw": "mocked_version)", + "exact_match": false, + "code_bleu": 0.23784142300054423, + "edit_similarity": 0.7317073170731707 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "parsed.url.query.params.items()", + "got": "parsed.query", + "got_raw": "parsed.query", + "exact_match": false, + "code_bleu": 0.1928473039599675, + "edit_similarity": 0.5581395348837209 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "1)", + "got": "=", + "got_raw": " = ProjectMock(tmp_path)", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "expected_toml)", + "got": "ppath=tmp_path)", + "got_raw": "ppath=tmp_path)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.3448275862068966 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"\"\" [section]", + "got": "= ProjectMock(tmp_path)", + "got_raw": " = ProjectMock(tmp_path)", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.2777777777777778 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"\"\" [something] x = 1", + "got": "= ProjectMock(tmp_path)", + "got_raw": " = ProjectMock(tmp_path)", + "exact_match": false, + "code_bleu": 0.17063255391931667, + "edit_similarity": 0.09090909090909091 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "content)", + "got": "content)", + "got_raw": "content)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "from_path", + "got": "from_path", + "got_raw": "from_path", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "dedent(after)", + "got": "dedent(after)", + "got_raw": "dedent(after)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"\"\" [my.table]", + "got": "dedent(\"\"\"", + "got_raw": "dedent(\"\"\"", + "exact_match": false, + "code_bleu": 0.2702118882866216, + "edit_similarity": 0.25 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"\"\" [my.table]", + "got": "\"\")", + "got_raw": "\"\")", + "exact_match": false, + "code_bleu": 0.153509990260506, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"\"\" [my.table]", + "got": "\"my.table\\n\\n[my.table]\\nkey = value\\n\\n[my.table]\\nkey2 = value2\")", + "got_raw": "\"my.table\\n\\n[my.table]\\nkey = value\\n\\n[my.table]\\nkey2 = value2\")", + "exact_match": false, + "code_bleu": 0.1290860005156921, + "edit_similarity": 0.2716049382716049 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"\"\" [my.table] key-before = 3", + "got": "\"\"\")", + "got_raw": "\"\"\")", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"\"\" [my.table]", + "got": "\"\")", + "got_raw": "\"\")", + "exact_match": false, + "code_bleu": 0.153509990260506, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"\"\" [my.table]", + "got": "\"", + "got_raw": "\"", + "exact_match": false, + "code_bleu": 0.08494336535733721, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"\"\" [my.table]", + "got": "\"my.table\\n\\n[my.table]\\n\\n[my.table]\\n\\n[my.table]\")", + "got_raw": "\"my.table\\n\\n[my.table]\\n\\n[my.table]\\n\\n[my.table]\")", + "exact_match": false, + "code_bleu": 0.1290860005156921, + "edit_similarity": 0.3283582089552239 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"\"\" [my.table]", + "got": "\"\")", + "got_raw": "\"\")", + "exact_match": false, + "code_bleu": 0.153509990260506, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "0", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "expected_yaml)", + "got": "og=True)", + "got_raw": "og=True)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "datadir / \"existing-expected.yaml\")", + "got": "-expected.yaml\")", + "got_raw": "-expected.yaml\")", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.6274509803921569 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "datadir / \"list-by-hash-expected.yaml\")", + "got": "objects.yaml\")", + "got_raw": "objects.yaml\")", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.41509433962264153 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "datadir / \"jmes-list-key-expected.yaml\")", + "got": "st-key.yaml\")", + "got_raw": "st-key.yaml\")", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.49056603773584906 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "datadir / \"any-order.yaml\")", + "got": ".yaml\")", + "got_raw": ".yaml\")", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.4117647058823529 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "datadir / \"wildcard-expected.yaml\")", + "got": "ppath=tmp_path)", + "got_raw": "ppath=tmp_path)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.2 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "datadir / \"same-key-expected.yaml\")", + "got": "ppath=tmp_path)", + "got_raw": "ppath=tmp_path)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.2 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "datadir / \"dict-search-by-key-expected.yaml\")", + "got": "ppath=tmp_path)", + "got_raw": "ppath=tmp_path)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.2 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "datadir / \"scalar-add-elements-that-do-not-exist-expected.yaml\")", + "got": "ppath=tmp_path)", + "got_raw": "ppath=tmp_path)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.1518987341772152 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "datadir / \"2-untouched-pre-commit.yaml\")", + "got": "", + "got_raw": "", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"\"\" repos: [] \"\"\")", + "got": "= ProjectMock(tmp_path)", + "got_raw": " = ProjectMock(tmp_path)", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.1951219512195122 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "\"\"\" repos: - hooks: - id: whatever - grepo: glocal \"\"\")", + "got": "= ProjectMock(tmp_path)", + "got_raw": " = ProjectMock(tmp_path)", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.20512820512820512 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "r\"\"\" repos: - repo: another hooks: - id: isort - repo: another hooks: \"- name: isort\\n entry: isort -sp setup.cfg\\n\" \"\"\")", + "got": "= ProjectMock(tmp_path)", + "got_raw": " = ProjectMock(tmp_path)", + "exact_match": false, + "code_bleu": 0.06204820142520606, + "edit_similarity": 0.125 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "datadir / \"hook-args-add.yaml\")", + "got": "uss(Fuss(\"missing key\"", + "got_raw": "uss(Fuss(\"missing key\", \"missing value\"))", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.07547169811320754 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "datadir / \"hook-args-change.yaml\")", + "got": "uss(Fuss(SharedViolations.PRE_COMMIT_CONFIG_YAML))", + "got_raw": "uss(Fuss(SharedViolations.PRE_COMMIT_CONFIG_YAML))", + "exact_match": false, + "code_bleu": 0.14113991930789777, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "datadir / \"uk-empty-expected.yaml\")", + "got": "uss(Fuss(key=\"list\"", + "got_raw": "uss(Fuss(key=\"list\", value=\"\"))", + "exact_match": false, + "code_bleu": 0.1561969968460128, + "edit_similarity": 0.18518518518518517 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "datadir / \"uk-default-expected.yaml\")", + "got": "uss(Fuss(PRE_COMMIT_CONFIG_YAML", + "got_raw": "uss(Fuss(PRE_COMMIT_CONFIG_YAML, \"repo: git@github.com:foo/bar\"))", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.058823529411764705 + }, + { + "repo": "andreoliwa/nitpick", + "expected": "datadir / \"uk-override-expected.yaml\")", + "got": "uss(Fuss(\"foo\"", + "got_raw": "uss(Fuss(\"foo\", \"bar\"))", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.07692307692307693 + }, + { + "repo": "andrewyng/aisuite", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "6", + "got": "5", + "got_raw": "5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "5)", + "got": "5)", + "got_raw": "5)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "20", + "got": "20", + "got_raw": "20", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "30", + "got": "30", + "got_raw": "30", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "20", + "got": "20", + "got_raw": "20", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "30", + "got": "30", + "got_raw": "30", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "20", + "got": "20", + "got_raw": "20", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "30", + "got": "30", + "got_raw": "30", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "20", + "got": "20", + "got_raw": "20", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "30", + "got": "30", + "got_raw": "30", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "20", + "got": "20", + "got_raw": "20", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "30", + "got": "30", + "got_raw": "30", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "2)", + "got": "True)", + "got_raw": "True)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "andrewyng/aisuite", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "0.9", + "got": "0.9", + "got_raw": "0.9", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "0.5", + "got": "0.5", + "got_raw": "0.5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "0.0", + "got": "0.0", + "got_raw": "0.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "0.5", + "got": "0.5", + "got_raw": "0.5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "0.9", + "got": "0.9", + "got_raw": "0.9", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "10)", + "got": "10)", + "got_raw": "10)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "15)", + "got": "15)", + "got_raw": "15)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "20)", + "got": "20)", + "got_raw": "20)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "30)", + "got": "30)", + "got_raw": "30)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "0.7", + "got": "0.7", + "got_raw": "0.7", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"en\"", + "got": "\"en\"", + "got_raw": "\"en\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"en\"", + "got": "\"en\"", + "got_raw": "\"en\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"en\"", + "got": "\"en\"", + "got_raw": "\"en\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"en\"", + "got": "\"en\"", + "got_raw": "\"en\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"en\"", + "got": "\"en\"", + "got_raw": "\"en\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"72\")", + "got": "\"72\")", + "got_raw": "\"72\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"warn\"", + "got": "\"warn\"", + "got_raw": "\"warn\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"Test\"", + "got": "mock_result.text", + "got_raw": "mock_result.text", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.2727272727272727 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"fast\"", + "got": "{\"content\":", + "got_raw": "{\"content\": [{\"type\": \"text\", \"text\": \"fast\"}]}", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"slow\"", + "got": "\"slow\"", + "got_raw": "\"slow\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"json\"", + "got": "\"json\"", + "got_raw": "\"json\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"user\"", + "got": "\"USER\"", + "got_raw": "\"USER\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"stop\"", + "got": "\"stop\"", + "got_raw": "\"stop\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"true\"", + "got": "\"true\"", + "got_raw": "\"true\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"Test\"", + "got": "\"Test\"", + "got_raw": "\"Test\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"tool1\"", + "got": "\"tool1\"", + "got_raw": "\"tool1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"tool2\"", + "got": "\"tool2\"", + "got_raw": "\"tool2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"stop\")", + "got": "\"end_turn\")", + "got_raw": "\"end_turn\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"user\")", + "got": "\"tool\")", + "got_raw": "\"tool\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.42857142857142855 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"en-US\"", + "got": "\"en-US\"", + "got_raw": "\"en-US\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"user\")", + "got": "\"user\")", + "got_raw": "\"user\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"stop\")", + "got": "\"complete\")", + "got_raw": "\"complete\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"en-US\"", + "got": "\"en-US\"", + "got_raw": "\"en-US\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"user\")", + "got": "\"USER\")", + "got_raw": "\"USER\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.42857142857142855 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"stop\")", + "got": "\"stop\")", + "got_raw": "\"stop\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"model\"", + "got": "\"assistant\"", + "got_raw": "\"assistant\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"Hello\"", + "got": "\"Hello\"", + "got_raw": "\"Hello\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"model\")", + "got": "\"assistant\")", + "got_raw": "\"assistant\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"Hello!\"", + "got": "\"Hello!\"", + "got_raw": "\"Hello!\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\" Hello\"", + "got": "\"Hello\"", + "got_raw": "\"Hello\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"array\")", + "got": "\"array\")", + "got_raw": "\"array\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"array\")", + "got": "input_schema)", + "got_raw": "input_schema)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"email\")", + "got": "\"email\"", + "got_raw": "\"email\", tool_spec[\"parameters\"][\"properties\"][\"email\"][\"format\"])", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"object\")", + "got": "\"schema\")", + "got_raw": "\"schema\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"object\")", + "got": "\"object\")", + "got_raw": "\"object\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"string\")", + "got": "\"string\")", + "got_raw": "\"string\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"tool123\")", + "got": "normalized_response.choices[0].message.tool_calls[0].type", + "got_raw": "normalized_response.choices[0].message.tool_calls[0].type, \"tool_use\")", + "exact_match": false, + "code_bleu": 0.06917184228205472, + "edit_similarity": 0.11940298507462686 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"tool123\")", + "got": "\"tool123\")", + "got_raw": "\"tool123\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"function\"", + "got": "\"function\"", + "got_raw": "\"function\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"success\")", + "got": "\"success\")", + "got_raw": "\"success\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"Celsius\")", + "got": "\"Celsius\")", + "got_raw": "\"Celsius\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"read_file\"", + "got": "\"read_file\"", + "got_raw": "\"read_file\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"test_tool\"", + "got": "\"test_tool\"", + "got_raw": "\"test_tool\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"top_song\")", + "got": "\"top_song\")", + "got_raw": "\"top_song\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"function\")", + "got": "\"function\")", + "got_raw": "\"function\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"audio/wav\"", + "got": "\"audio/wav\"", + "got_raw": "\"audio/wav\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"SSE result\"", + "got": "\"SSE result\"", + "got_raw": "\"SSE result\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"assistant\")", + "got": "\"assistant\")", + "got_raw": "\"assistant\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"assistant\")", + "got": "\"assistant\")", + "got_raw": "\"assistant\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"Hello world\"", + "got": "\"Hello world\"", + "got_raw": "\"Hello world\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"test-server\"", + "got": "\"test-server\"", + "got_raw": "\"test-server\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"tool_calls\")", + "got": "\"tool_use\")", + "got_raw": "\"tool_use\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.75 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"tool_calls\")", + "got": "\"function_call\")", + "got_raw": "\"function_call\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6896551724137931 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"Hello world\"", + "got": "\"Hello world\"", + "got_raw": "\"Hello world\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"final result\"", + "got": "{\"content\": [{\"type\": \"text\"", + "got_raw": "{\"content\": [{\"type\": \"text\", \"text\": \"final result\"}]}", + "exact_match": false, + "code_bleu": 0.13485111859503685, + "edit_similarity": 0.23809523809523808 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"get_weather\")", + "got": "\"get_weather\")", + "got_raw": "\"get_weather\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"verbose_json\"", + "got": "\"verbose_json\"", + "got_raw": "\"verbose_json\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"get_weather\")", + "got": "\"get_weather\")", + "got_raw": "\"get_weather\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"test-api-key\"", + "got": "\"test-api-key\"", + "got_raw": "\"test-api-key\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"get_weather\")", + "got": "\"get_weather\")", + "got_raw": "\"get_weather\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"test_simple\")", + "got": "\"test_simple\")", + "got_raw": "\"test_simple\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"Hello, World!\"", + "got": "\"Hello\"", + "got_raw": "\"Hello\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6363636363636364 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"test-hf-token\"", + "got": "\"test-hf-token\"", + "got_raw": "\"test-hf-token\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"San Francisco\")", + "got": "\"San Francisco\")", + "got_raw": "\"San Francisco\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"Mixed tools work!\"", + "got": "\"Mixed tools work!\"", + "got_raw": "\"Mixed tools work!\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"Get the weather.\")", + "got": "\"Get the weather.\")", + "got_raw": "\"Get the weather.\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "'{\"sign\": \"WZPZ\"}')", + "got": "{\"sign\": \"WZPZ\"})", + "got_raw": "{\"sign\": \"WZPZ\"})", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.9444444444444444 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"Test error message\"", + "got": "\"Test error message\"", + "got_raw": "\"Test error message\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "\"get_exchange_rate\")", + "got": "len(normalized_response.choices[0].tool_calls)", + "got_raw": "len(normalized_response.choices[0].tool_calls)", + "exact_match": false, + "code_bleu": 0.11390778025531027, + "edit_similarity": 0.21212121212121213 + }, + { + "repo": "andrewyng/aisuite", + "expected": "tool", + "got": "tool", + "got_raw": "tool", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "content", + "got": "content", + "got_raw": "content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "headers", + "got": "headers", + "got_raw": "headers", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "ASRError)", + "got": "ASRError)", + "got_raw": "ASRError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "Exception)", + "got": "Exception)", + "got_raw": "Exception)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "tool_names", + "got": "tool_names", + "got_raw": "tool_names", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "tool_names", + "got": "tool_names", + "got_raw": "tool_names", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "audio_data", + "got": "audio_data", + "got_raw": "audio_data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "call_kwargs", + "got": "call_kwargs", + "got_raw": "call_kwargs", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "tool_names)", + "got": "tool_names)", + "got_raw": "tool_names)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "schema_json)", + "got": "schema_json)", + "got_raw": "schema_json)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "schema_copy)", + "got": "schema_copy)", + "got_raw": "schema_copy)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "text_content)", + "got": "text_content)", + "got_raw": "text_content)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "input_schema)", + "got": "input_schema)", + "got_raw": "input_schema)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "original_error", + "got": "original_error", + "got_raw": "original_error", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "selected_model", + "got": "selected_model", + "got_raw": "selected_model", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "PROVIDER_PARAMS", + "got": "PROVIDER_PARAMS", + "got_raw": "PROVIDER_PARAMS", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "expected_tool_spec", + "got": "expected_tool_spec", + "got_raw": "expected_tool_spec", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "GOOGLE_LANGUAGE_MAP", + "got": "GOOGLE_LANGUAGE_MAP", + "got_raw": "GOOGLE_LANGUAGE_MAP", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "response_text_content", + "got": "response_text_content", + "got_raw": "response_text_content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "response_text_content", + "got": "response_text_content", + "got_raw": "response_text_content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "response_text_content", + "got": "response_text_content", + "got_raw": "response_text_content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "response_text_content", + "got": "response_text_content", + "got_raw": "response_text_content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "expected_content_type", + "got": "expected_content_type", + "got_raw": "expected_content_type", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "response_text_content", + "got": "response_text_content", + "got_raw": "response_text_content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "response_text_content", + "got": "response_text_content", + "got_raw": "response_text_content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "response_text_content", + "got": "response_text_content", + "got_raw": "response_text_content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "response_text_content", + "got": "response_text_content", + "got_raw": "response_text_content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "results[0])", + "got": "results[0])", + "got_raw": "results[0])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "provider.api_key", + "got": "provider.api_key", + "got_raw": "provider.api_key", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "call_args.args[0]", + "got": "call_args[0][0]", + "got_raw": "call_args[0][0]", + "exact_match": false, + "code_bleu": 0.3655552228545123, + "edit_similarity": 0.75 + }, + { + "repo": "andrewyng/aisuite", + "expected": "provider.project_id", + "got": "\"test-project-id\"", + "got_raw": "\"test-project-id\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "andrewyng/aisuite", + "expected": "provider.service_url", + "got": "provider.service_url", + "got_raw": "provider.service_url", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "COMMON_PARAMS[\"language\"]", + "got": "COMMON_PARAMS", + "got_raw": "COMMON_PARAMS", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.6842105263157895 + }, + { + "repo": "andrewyng/aisuite", + "expected": "ValueError, match=\"Test error\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5238095238095238 + }, + { + "repo": "andrewyng/aisuite", + "expected": "ImportError, match=\"mcp.*package\")", + "got": "ImportError)", + "got_raw": "ImportError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5217391304347826 + }, + { + "repo": "andrewyng/aisuite", + "expected": "ASRError, match=\"Specific ASR error\")", + "got": "ASRError)", + "got_raw": "ASRError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.391304347826087 + }, + { + "repo": "andrewyng/aisuite", + "expected": "ValueError, match=\"must have 'name'\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.4583333333333333 + }, + { + "repo": "andrewyng/aisuite", + "expected": "response.choices[0].message.tool_calls)", + "got": "response.usage)", + "got_raw": "response.usage)", + "exact_match": false, + "code_bleu": 0.18477554478349628, + "edit_similarity": 0.5185185185185185 + }, + { + "repo": "andrewyng/aisuite", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "{}", + "got": "{\"temperature\":", + "got_raw": "{\"temperature\": 0.5}", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "andrewyng/aisuite", + "expected": "[])", + "got": "\"User:", + "got_raw": "\"User: Hello, how are you?\")", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "[])", + "got": "\"User:", + "got_raw": "\"User: What is the most popular song on WZPZ?\")", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "{})", + "got": "{})", + "got_raw": "{})", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "{\"diarize\": True}", + "got": "{\"diarize\": True}", + "got_raw": "{\"diarize\": True}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "{\"language\": \"en\"}", + "got": "{\"language\": \"en\"}", + "got_raw": "{\"language\": \"en\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "{\"language\": None}", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.23784142300054423, + "edit_similarity": 0.2 + }, + { + "repo": "andrewyng/aisuite", + "expected": "{\"punctuate\": True}", + "got": "{\"punctuate\": True}", + "got_raw": "{\"punctuate\": True}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "[\"technical terms\"]", + "got": "[\"technical terms\"]", + "got_raw": "[\"technical terms\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "[expected_message])", + "got": "[expected_message])", + "got_raw": "[expected_message])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "[\"required_param\"])", + "got": "[\"required_param\"])", + "got_raw": "[\"required_param\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "{\"temperature\": 0.5}", + "got": "{\"temperature\": 0.5}", + "got_raw": "{\"temperature\": 0.5}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "andrewyng/aisuite", + "expected": "{\"unknown_param\": True}", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.23784142300054423, + "edit_similarity": 0.16 + }, + { + "repo": "andrewyng/aisuite", + "expected": "{\"keywords\": [\"meeting\"]}", + "got": "{\"prompt\": [\"meeting\"]}", + "got_raw": "{\"prompt\": [\"meeting\"]}", + "exact_match": false, + "code_bleu": 0.7071067811865475, + "edit_similarity": 0.75 + }, + { + "repo": "andrewyng/aisuite", + "expected": "{\"temperature\": \"15\", \"unit\": \"Celsius\"})", + "got": "Function)", + "got_raw": "Function)", + "exact_match": false, + "code_bleu": 0.1074569931823542, + "edit_similarity": 0.2 + }, + { + "repo": "andrewyng/aisuite", + "expected": "str(context.exception))", + "got": "\"Invalid", + "got_raw": "\"Invalid type for 'location' parameter\")", + "exact_match": false, + "code_bleu": 0.1290860005156921, + "edit_similarity": 0.12903225806451613 + }, + { + "repo": "claffin/cloudproxy", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "422", + "got": "400", + "got_raw": "400", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "claffin/cloudproxy", + "expected": "2", + "got": "5", + "got_raw": "5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "True", + "got": "False", + "got_raw": "False", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "claffin/cloudproxy", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"192.168.1.1\"", + "got": "\"192.168.1.1\"", + "got_raw": "\"192.168.1.1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "custom_session", + "got": "custom_session", + "got_raw": "custom_session", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"https://api.ipify.org\")", + "got": "_session.get.assert_called_once_with(\"http://10.0.0.1:8899\")", + "got_raw": "_session.get.assert_called_once_with(\"http://10.0.0.1:8899\")", + "exact_match": false, + "code_bleu": 0.16036590969929357, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"http://ipecho.net/plain\")", + "got": "_get.assert_called_once_with(\"http://10.0.0.1\")", + "got_raw": "_get.assert_called_once_with(\"http://10.0.0.1\")", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.3013698630136986 + }, + { + "repo": "claffin/cloudproxy", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"mocked-ip-response\"", + "got": "proxy", + "got_raw": "proxy", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.24 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"https://api.ipify.org\")", + "got": "_session.get.assert_called_with(\"https://ipinfo.io/ip\"", + "got_raw": "_session.get.assert_called_with(\"https://ipinfo.io/ip\", proxies=expected_proxies)", + "exact_match": false, + "code_bleu": 0.1561969968460128, + "edit_similarity": 0.38461538461538464 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"http://ipecho.net/plain\")", + "got": "_requests_get.assert_called_with(f\"https://ipapi.co/{proxy['ip']}/json\"", + "got_raw": "_requests_get.assert_called_with(f\"https://ipapi.co/{proxy['ip']}/json\", proxies=expected_proxy)", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.28865979381443296 + }, + { + "repo": "claffin/cloudproxy", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "404", + "got": "200", + "got_raw": "200", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "claffin/cloudproxy", + "expected": "data", + "got": "data", + "got_raw": "data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "5", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "422", + "got": "400", + "got_raw": "400", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "claffin/cloudproxy", + "expected": "404", + "got": "404", + "got_raw": "404", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "data", + "got": "data", + "got_raw": "data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "8000", + "got": "8000", + "got_raw": "8000", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "schema", + "got": "schema", + "got_raw": "schema", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "6", + "got": "6", + "got_raw": "6", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "tags", + "got": "[{\"Key\":", + "got_raw": "[{\"Key\": \"Name\", \"Value\": \"CloudProxy-Test Instance\"}]", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"ec2\")", + "got": "_boto3_resource.assert_called_with(\"ec2\"", + "got_raw": "_boto3_resource.assert_called_with(\"ec2\", region_name=\"us-west-2\")", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.21739130434782608 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"spot\"", + "got": "\"spot\"", + "got_raw": "\"spot\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"stop\"", + "got": "\"terminate\"", + "got_raw": "\"terminate\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"instance\"", + "got": "\"instance\"", + "got_raw": "\"instance\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"persistent\"", + "got": "\"persistent\"", + "got_raw": "\"persistent\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"mock-client\"", + "got": "\"mock-client\"", + "got_raw": "\"mock-client\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"mock-resource\"", + "got": "\"mock-resource\"", + "got_raw": "\"mock-resource\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "[mock_instance]", + "got": "\"i-12345\"", + "got_raw": "\"i-12345\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.08333333333333333 + }, + { + "repo": "claffin/cloudproxy", + "expected": "stopped_instances", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "claffin/cloudproxy", + "expected": "started_instances", + "got": "\"i-12345\"", + "got_raw": "\"i-12345\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.07692307692307693 + }, + { + "repo": "claffin/cloudproxy", + "expected": "terminated_instances", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "claffin/cloudproxy", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "filedata", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "claffin/cloudproxy", + "expected": "Exception)", + "got": "Exception)", + "got_raw": "Exception)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "str(exc_info.value)", + "got": "str(exc_info.value)", + "got_raw": "str(exc_info.value)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "DOFirewallExistsException)", + "got": "DOFirewallExistsException)", + "got_raw": "DOFirewallExistsException)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "True", + "got": "droplet", + "got_raw": "droplet", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "claffin/cloudproxy", + "expected": "1234)", + "got": "1234)", + "got_raw": "1234)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"nyc1\"", + "got": "\"useast\"", + "got_raw": "\"useast\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "claffin/cloudproxy", + "expected": "3164444", + "got": "droplets[0].id", + "got_raw": "droplets[0].id", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "result_ids", + "got": "result_ids", + "got_raw": "result_ids", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "droplet_id)", + "got": "droplet_id)", + "got_raw": "droplet_id)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "'cloudproxy'", + "got": "'cloudproxy'", + "got_raw": "'cloudproxy'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "expected_ids", + "got": "expected_ids", + "got_raw": "expected_ids", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"s-1vcpu-1gb\"", + "got": "\"512mb\"", + "got_raw": "\"512mb\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "claffin/cloudproxy", + "expected": "kwargs['tags']", + "got": "args[\"tags\"]", + "got_raw": "args[\"tags\"]", + "exact_match": false, + "code_bleu": 0.3976353643835253, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "claffin/cloudproxy", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "12345)", + "got": "12345)", + "got_raw": "12345)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "Exception, match=\"Some other error\")", + "got": "Exception)", + "got_raw": "Exception)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.43478260869565216 + }, + { + "repo": "claffin/cloudproxy", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "3164444", + "got": "droplets[0].id", + "got_raw": "droplets[0].id", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "droplet_id)", + "got": "droplet_id)", + "got_raw": "droplet_id)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "[\"192.1.1.1\"]", + "got": "[2]", + "got_raw": "[2]", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.375 + }, + { + "repo": "claffin/cloudproxy", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "0", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "1)", + "got": "3)", + "got_raw": "3)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "claffin/cloudproxy", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "[])", + "got": "False)", + "got_raw": "False)", + "exact_match": false, + "code_bleu": 0.35818997727451396, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "claffin/cloudproxy", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "None", + "got": "False", + "got_raw": "False", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "claffin/cloudproxy", + "expected": "body[\"name\"]", + "got": "body[\"name\"]", + "got_raw": "body[\"name\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"cloudproxy\"", + "got": "\"true\"", + "got_raw": "\"true\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"ONE_TO_ONE_NAT\"", + "got": "\"ONE_TO_ONE_NAT\"", + "got_raw": "\"ONE_TO_ONE_NAT\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"cloudproxy-123\"", + "got": "\"cloudproxy-123\"", + "got_raw": "\"cloudproxy-123\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "body[\"tags\"][\"items\"]", + "got": "body[\"labels\"]", + "got_raw": "body[\"labels\"]", + "exact_match": false, + "code_bleu": 0.29036157037406557, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "claffin/cloudproxy", + "expected": "{\"status\": \"RUNNING\"}", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.08 + }, + { + "repo": "claffin/cloudproxy", + "expected": "{\"status\": \"STOPPING\"}", + "got": "\"STOPPING\"", + "got_raw": "\"STOPPING\"", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.625 + }, + { + "repo": "claffin/cloudproxy", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "3", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"hel1\"", + "got": "\"europe\"", + "got_raw": "\"europe\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.42857142857142855 + }, + { + "repo": "claffin/cloudproxy", + "expected": "result_ids", + "got": "result_ids", + "got_raw": "result_ids", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "mock_response", + "got": "\"test-uuid\"", + "got_raw": "\"test-uuid\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"server-id-1\")", + "got": "\"server-id-1\")", + "got_raw": "\"server-id-1\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"delete-response\"", + "got": "\"delete-response\"", + "got_raw": "\"delete-response\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"type=cloudproxy\"", + "got": "\"type=cloudproxy\"", + "got_raw": "\"type=cloudproxy\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "mock_client_instance", + "got": "mock_client_instance", + "got_raw": "mock_client_instance", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "[])", + "got": "[\"1.1.1.1\"])", + "got_raw": "[\"1.1.1.1\"])", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.4 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"1.1.1.1\")", + "got": "mock_proxy)", + "got_raw": "mock_proxy)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "claffin/cloudproxy", + "expected": "[\"1.1.1.1\"])", + "got": "[\"1.1.1.1\"])", + "got_raw": "[\"1.1.1.1\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "mock_delete_queue)", + "got": "mock_delete_queue)", + "got_raw": "mock_delete_queue)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "mock_restart_queue)", + "got": "mock_restart_queue)", + "got_raw": "mock_restart_queue)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "mock_instance_config)", + "got": "mock_instance_config)", + "got_raw": "mock_instance_config)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "delete_queue", + "got": "delete_queue", + "got_raw": "delete_queue", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "delete_queue)", + "got": "delete_queue)", + "got_raw": "delete_queue)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "4", + "got": "5", + "got_raw": "5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "functions", + "got": "functions", + "got_raw": "functions", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "Exception)", + "got": "Exception)", + "got_raw": "Exception)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "expected_ips", + "got": "expected_ips", + "got_raw": "expected_ips", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "test_config)", + "got": "test_config)", + "got_raw": "test_config)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "config", + "got": "config", + "got_raw": "config", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"t3.micro\"", + "got": "\"t3.micro\"", + "got_raw": "\"t3.micro\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"t2.small\"", + "got": "\"t2.small\"", + "got_raw": "\"t2.small\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"t2.large\"", + "got": "\"t2.large\"", + "got_raw": "\"t2.large\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"us-west1-b\"", + "got": "\"us-west1-b\"", + "got_raw": "\"us-west1-b\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"test-project\"", + "got": "\"test-project\"", + "got_raw": "\"test-project\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "config[\"providers\"]", + "got": "config[\"providers\"]", + "got_raw": "config[\"providers\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "config[\"providers\"][\"aws\"][\"instances\"][\"default\"][\"size\"]", + "got": "config[\"providers\"][\"aws\"][\"instances\"][\"default\"][\"size\"]", + "got_raw": "config[\"providers\"][\"aws\"][\"instances\"][\"default\"][\"size\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "config[\"providers\"][\"aws\"][\"instances\"][\"default\"][\"enabled\"]", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.03125, + "edit_similarity": 0.09230769230769231 + }, + { + "repo": "claffin/cloudproxy", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "387", + "got": "387", + "got_raw": "387", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"ewr\"", + "got": "\"ewr\"", + "got_raw": "\"ewr\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"tcp\"", + "got": "\"tcp\"", + "got_raw": "\"tcp\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"udp\"", + "got": "\"udp\"", + "got_raw": "\"udp\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"8899\"", + "got": "8899", + "got_raw": "8899", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"active\"", + "got": "\"active\"", + "got_raw": "\"active\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"test-id\"", + "got": "\"test-id\"", + "got_raw": "\"test-id\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"testpass\")", + "got": "\"testpass\")", + "got_raw": "\"testpass\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"test-label\"", + "got": "\"test-label\"", + "got_raw": "\"test-label\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"vc2-1c-1gb\"", + "got": "\"vc2-1c-1gb\"", + "got_raw": "\"vc2-1c-1gb\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "1", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "3", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"192.168.1.1\")", + "got": "\"192.168.1.1\")", + "got_raw": "\"192.168.1.1\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "8899", + "got": "8899", + "got_raw": "8899", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "2", + "got": "\"192.168.1.1", + "got_raw": "\"192.168.1.1,192.168.1.2\"", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "claffin/cloudproxy", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "now", + "got": "now", + "got_raw": "now", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "3600", + "got": "3600", + "got_raw": "3600", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "\"aws\"", + "got": "\"aws\"", + "got_raw": "\"aws\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "status", + "got": "status", + "got_raw": "status", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "state.pending", + "got": "state.pending", + "got_raw": "state.pending", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "claffin/cloudproxy", + "expected": "content", + "got": "content", + "got_raw": "content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "tx)", + "got": "tx)", + "got_raw": "tx)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "slug", + "got": "slug", + "got_raw": "slug", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "tags", + "got": "tags", + "got_raw": "tags", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "msg)", + "got": "msg)", + "got_raw": "msg)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "list)", + "got": "list)", + "got_raw": "list)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "dict)", + "got": "dict)", + "got_raw": "dict)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "func)", + "got": "func)", + "got_raw": "func)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "event", + "got": "event", + "got_raw": "event", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "kwargs", + "got": "kwargs", + "got_raw": "kwargs", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "amount", + "got": "amount", + "got_raw": "amount", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "found)", + "got": "found)", + "got_raw": "found)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "agent)", + "got": "agent)", + "got_raw": "agent)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "events", + "got": "events", + "got_raw": "events", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "states", + "got": "states", + "got_raw": "states", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "config", + "got": "config", + "got_raw": "config", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "chains)", + "got": "chains)", + "got_raw": "chains)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "prices)", + "got": "prices)", + "got_raw": "prices)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "post_id", + "got": "post_id", + "got_raw": "post_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "profile", + "got": "profile", + "got_raw": "profile", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "user_id", + "got": "user_id", + "got_raw": "user_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "headers", + "got": "headers", + "got_raw": "headers", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "returns", + "got": "returns", + "got_raw": "returns", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "token_a", + "got": "token_a", + "got_raw": "token_a", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "token_b", + "got": "token_b", + "got_raw": "token_b", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "address2", + "got": "address2", + "got_raw": "address2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "address3", + "got": "address3", + "got_raw": "address3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "address4", + "got": "address4", + "got_raw": "address4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "next_run", + "got": "next_run", + "got_raw": "next_run", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "agent_id", + "got": "user_id", + "got_raw": "user_id", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "account)", + "got": "account)", + "got_raw": "account)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "coin_data)", + "got": "coin_data)", + "got_raw": "coin_data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "fake_bytes", + "got": "fake_bytes", + "got_raw": "fake_bytes", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "mock_event", + "got": "mock_event", + "got_raw": "mock_event", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "[events[0]]", + "got": "[events[0]]", + "got_raw": "[events[0]]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "[\"img1.jpg\"]", + "got": "[\"img1.jpg\"]", + "got_raw": "[\"img1.jpg\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "{\"chain\": chain})", + "got": "dummy.json)", + "got_raw": "dummy.json)", + "exact_match": false, + "code_bleu": 0.23953607464205617, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "{\"fees\": \"data\"})", + "got": "dummy.json()[\"fees\"])", + "got_raw": "dummy.json()[\"fees\"])", + "exact_match": false, + "code_bleu": 0.15619699684601276, + "edit_similarity": 0.3684210526315789 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "{\"protocols\": []})", + "got": "[])", + "got_raw": "[])", + "exact_match": false, + "code_bleu": 0.27383275896268816, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "3", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "4", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "4", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "0", + "got": "Decimal(\"1000000000000000000\")", + "got_raw": "Decimal(\"1000000000000000000\")", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.06451612903225806 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "0", + "got": "999", + "got_raw": "999", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"0\"", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"0\"", + "got": "\"0.000000\"", + "got_raw": "\"0.000000\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"add\"", + "got": "\"add\"", + "got_raw": "\"add\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"low\"", + "got": "\"100\"", + "got_raw": "\"100\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"ABC\"", + "got": "\"ABC\"", + "got_raw": "\"ABC\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"XYZ\"", + "got": "\"XYZ\"", + "got_raw": "\"XYZ\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"app\"", + "got": "\"app\"", + "got_raw": "\"app\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"safe\"", + "got": "\"privy\"", + "got_raw": "\"privy\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"KEEP\"", + "got": "\"KEEP\"", + "got_raw": "\"KEEP\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"POST\"", + "got": "\"POST\"", + "got_raw": "\"POST\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"USDC\"", + "got": "\"USDC\"", + "got_raw": "\"USDC\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"1.23\"", + "got": "\"1.23\"", + "got_raw": "\"1.23\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"0abc\"", + "got": "\"0abc\"", + "got_raw": "\"0abc\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"0def\"", + "got": "\"0xdef\"", + "got_raw": "\"0xdef\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"TRADE\"", + "got": "\"TRADE\"", + "got_raw": "\"TRADE\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"plain\"", + "got": "\"plain\"", + "got_raw": "\"plain\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"task1\"", + "got": "\"task1\"", + "got_raw": "\"task1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"task2\"", + "got": "\"task2\"", + "got_raw": "\"task2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"error\"", + "got": "\"ERROR\"", + "got_raw": "\"ERROR\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"erc20\"", + "got": "\"erc20\"", + "got_raw": "\"erc20\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"tests\"", + "got": "\"tests\"", + "got_raw": "\"tests\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"0xbbb\"", + "got": "\"0xbbb\"", + "got_raw": "\"0xbbb\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"0xabc\"", + "got": "\"0xabc\"", + "got_raw": "\"0xabc\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"0x123\"", + "got": "\"0x123\"", + "got_raw": "\"0x123\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"system\"", + "got": "\"system\"", + "got_raw": "\"system\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"post-1\"", + "got": "\"post-1\"", + "got_raw": "\"post-1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"123.45\"", + "got": "\"123.45\"", + "got_raw": "\"123.45\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"user_1\"", + "got": "\"user_1\"", + "got_raw": "\"user_1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"team_1\"", + "got": "\"team_1\"", + "got_raw": "\"team_1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"gpt-4o\"", + "got": "\"template\"", + "got_raw": "\"template\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"user_2\"", + "got": "\"user_2\"", + "got_raw": "\"user_2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"memory\"", + "got": "\"expense\"", + "got_raw": "\"expense\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"task-1\"", + "got": "\"task-1\"", + "got_raw": "\"task-1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"post-1\"", + "got": "post.id", + "got_raw": "post.id", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"failed\"", + "got": "\"failed\"", + "got_raw": "\"failed\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"0xabcd\"", + "got": "\"0x00abcd\"", + "got_raw": "\"0x00abcd\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"0xABCD\"", + "got": "\"0xABCD\"", + "got_raw": "\"0xABCD\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"signed\"", + "got": "\"signed\"", + "got_raw": "\"signed\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"unsafe\"", + "got": "\"unsafe\"", + "got_raw": "\"unsafe\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"agent-1\"", + "got": "activity_create.agent_id", + "got_raw": "activity_create.agent_id", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.30303030303030304 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"agent-1\"", + "got": "post_create.agent_id", + "got_raw": "post_create.agent_id", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.3448275862068966 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"DB Post\"", + "got": "\"DB Post\"", + "got_raw": "\"DB Post\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"gpt-3.5\"", + "got": "\"gpt-3.5\"", + "got_raw": "\"gpt-3.5\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"updated\"", + "got": "\"updated\"", + "got_raw": "\"updated\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"adjust\")", + "got": "\"user-1\")", + "got_raw": "\"user-1\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"event-3\"", + "got": "\"event-3\"", + "got_raw": "\"event-3\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"event-1\"", + "got": "\"event-1\"", + "got_raw": "\"event-1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"chat-1\")", + "got": "\"chat-1\")", + "got_raw": "\"chat-1\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"test123\"", + "got": "data[\"id\"]", + "got_raw": "data[\"id\"]", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.21052631578947367 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"waiting\"", + "got": "\"waiting\"", + "got_raw": "\"waiting\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"agent-3\"", + "got": "\"agent-3\"", + "got_raw": "\"agent-3\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"kq_test\"", + "got": "\"kq_test\"", + "got_raw": "\"kq_test\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"0xprivy\"", + "got": "\"0xprivy\"", + "got_raw": "\"0xprivy\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"New Name\"", + "got": "\"New Name\"", + "got_raw": "\"New Name\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"DB Agent\"", + "got": "\"DB Agent\"", + "got_raw": "\"DB Agent\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"post-123\"", + "got": "\"post-123\"", + "got_raw": "\"post-123\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"DB Agent\"", + "got": "\"DB Agent\"", + "got_raw": "\"DB Agent\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"dev-user\"", + "got": "Decimal(\"0.5000\")", + "got_raw": "Decimal(\"0.5000\")", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.14814814814814814 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"New Task\"", + "got": "\"New Task\"", + "got_raw": "\"New Task\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"NotFound\"", + "got": "\"POST_NOT_FOUND\"", + "got_raw": "\"POST_NOT_FOUND\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"1.234567\"", + "got": "\"1.23\"", + "got_raw": "\"1.23\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"wallet_1\"", + "got": "\"wallet_1\"", + "got_raw": "\"wallet_1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"wallet_2\"", + "got": "\"wallet_2\"", + "got_raw": "\"wallet_2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"0 * * * *\"", + "got": "\"0 * * * *\"", + "got_raw": "\"0 * * * *\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"video.mp4\"", + "got": "\"video.mp4\"", + "got_raw": "\"video.mp4\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"Test Post\"", + "got": "\"Test Post\"", + "got_raw": "\"Test Post\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"Test Task\"", + "got": "\"Test Task\"", + "got_raw": "\"Test Task\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"order-789\"", + "got": "\"order-789\"", + "got_raw": "\"order-789\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"wss://eth\"", + "got": "\"wss://eth\"", + "got_raw": "\"wss://eth\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"signed_tx\"", + "got": "\"signed_tx\"", + "got_raw": "\"signed_tx\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"Test Agent\"", + "got": "\"Test Agent\"", + "got_raw": "\"Test Agent\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"get-target\"", + "got": "\"get-target\"", + "got_raw": "\"get-target\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "\"Test Agent\"", + "got": "\"Test Agent\"", + "got_raw": "\"Test Agent\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "True", + "got": "False", + "got_raw": "False", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "str(sql1)", + "got": "sql1", + "got_raw": "sql1", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "str(sql2)", + "got": "str(sql2)", + "got_raw": "str(sql2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "str(sql3)", + "got": "str(sql3)", + "got_raw": "str(sql3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "len(agents)", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "Decimal(\"0\")", + "got": "result.budget", + "got_raw": "result.budget", + "exact_match": false, + "code_bleu": 0.2849383821193472, + "edit_similarity": 0.16 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "Decimal(\"0\")", + "got": "base_llm_amount", + "got_raw": "base_llm_amount", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.14814814814814814 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "Decimal(\"2\")", + "got": "Decimal(\"2\")", + "got_raw": "Decimal(\"2\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "Decimal(\"2.5\")", + "got": "Decimal(\"2.5\")", + "got_raw": "Decimal(\"2.5\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "str(exc.value)", + "got": "str(exc)", + "got_raw": "str(exc)", + "exact_match": false, + "code_bleu": 0.4714045207910316, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "len(\"tx-\") + 8", + "got": "8", + "got_raw": "8", + "exact_match": false, + "code_bleu": 0.09910059291689341, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "now.isoformat()", + "got": "now.isoformat()", + "got_raw": "now.isoformat()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "agent.id", + "got": "\"agent-1\"", + "got_raw": "\"agent-1\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.5882352941176471 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "agent.name", + "got": "agent.name", + "got_raw": "agent.name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "agent.owner", + "got": "agent.owner", + "got_raw": "agent.owner", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "agent.model", + "got": "agent.model", + "got_raw": "agent.model", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "agent.prompt", + "got": "agent.prompt", + "got_raw": "agent.prompt", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "result.budget", + "got": "result.budget", + "got_raw": "result.budget", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "0.1, abs=1.0)", + "got": "0.1, abs=0.01),", + "got_raw": "0.1, abs=0.01),\n cost=pytest.approx(0.1, abs=0.01),\n )", + "exact_match": false, + "code_bleu": 0.5410822690539396, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "OwnerType.USER", + "got": "OwnerType.DEVELOPER", + "got_raw": "OwnerType.DEVELOPER", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "Chain.Ethereum", + "got": "Chain.Ethereum", + "got_raw": "Chain.Ethereum", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "Chain.Arbitrum", + "got": "Chain.Arbitrum", + "got_raw": "Chain.Arbitrum", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "AuthorType.AGENT", + "got": "AuthorType.AGENT", + "got_raw": "AuthorType.AGENT", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crestalnetwork/intentkit", + "expected": "OwnerType.PLATFORM", + "got": "OwnerType.NONE", + "got_raw": "OwnerType.NONE", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6875 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "caplog.text", + "got": "\"CustomUserAdmin\"", + "got_raw": "\"CustomUserAdmin\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "\"/path/to/obj/\"", + "got": "\"/path/to/obj/\"", + "got_raw": "\"/path/to/obj/\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "response.content", + "got": "response.content", + "got_raw": "response.content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "\"/accounts/profile/\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "\"/custom/success/path/\"", + "got": "\"/custom/success/path/\"", + "got_raw": "\"/custom/success/path/\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "\"path\"", + "got": "\"path\"", + "got_raw": "\"path\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "\"/path\"", + "got": "\"/path\"", + "got_raw": "\"/path\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "hash(\"path\")", + "got": "hash(\"path\")", + "got_raw": "hash(\"path\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "\"http://path\"", + "got": "\"http://path\"", + "got_raw": "\"http://path\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "\"/static/path\"", + "got": "\"Asset(path)\"", + "got_raw": "\"Asset(path)\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5925925925925926 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "\"https://path\"", + "got": "\"https://path\"", + "got_raw": "\"https://path\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "\"Asset: 'path'\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8275862068965517 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "forms.Asset(\"path\")", + "got": "forms.Asset(\"path\")", + "got_raw": "forms.Asset(\"path\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "forms.Asset(\"other\")", + "got": "\"other\"", + "got_raw": "\"other\"", + "exact_match": false, + "code_bleu": 0.09910059291689341, + "edit_similarity": 0.5185185185185185 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "''", + "got": "\"ESM(path)\"", + "got_raw": "\"ESM(path)\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "\"21\"", + "got": "\"13\"", + "got_raw": "\"13\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "\"bob\"", + "got": "\"eve\"", + "got_raw": "\"eve\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "\"eve\"", + "got": "\"eve\"", + "got_raw": "\"eve\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "response", + "got": "response", + "got_raw": "response", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "b\"\"", + "got": "b\"\"", + "got_raw": "b\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "{\"request\": request})", + "got": "erto_string.assert_called_once_with(\"hijack/remote_user.html\")", + "got_raw": "erto_string.assert_called_once_with(\"hijack/remote_user.html\")", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.1927710843373494 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "b\"HIJACKED\"", + "got": "b\"HIJACKED\"", + "got_raw": "b\"HIJACKED\"", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6285714285714286 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "has_perm", + "got": "has_perm", + "got_raw": "has_perm", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "response.content", + "got": "response.content", + "got_raw": "response.content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "\"/\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "302", + "got": "302", + "got_raw": "302", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "403", + "got": "403", + "got_raw": "403", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "400", + "got": "400", + "got_raw": "400", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-hijack/django-hijack", + "expected": "405", + "got": "405", + "got_raw": "405", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "3", + "got": "\"0x1\"", + "got_raw": "\"0x1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "acct2", + "got": "acct2", + "got_raw": "acct2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "signed", + "got": "signed", + "got_raw": "signed", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "\"scrypt\"", + "got": "\"pbkdf2\"", + "got_raw": "\"pbkdf2\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.375 + }, + { + "repo": "ethereum/eth-account", + "expected": "\"pbkdf2\"", + "got": "\"pbkdf2\"", + "got_raw": "\"pbkdf2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "account2", + "got": "account2", + "got_raw": "account2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "signed[\"v\"]", + "got": "v", + "got_raw": "v", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "ethereum/eth-account", + "expected": "signed[\"r\"]", + "got": "signed[\"r\"]", + "got_raw": "signed[\"r\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "128", + "got": "len(tx.blob_data.commitments)", + "got_raw": "len(tx.blob_data.commitments)", + "exact_match": false, + "code_bleu": 0.13485111859503685, + "edit_similarity": 0.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "ZERO_BLOB", + "got": "ZERO_BLOB", + "got_raw": "ZERO_BLOB", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "blob_data_1", + "got": "blob_data_1", + "got_raw": "blob_data_1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "expected_blob", + "got": "expected_blob", + "got_raw": "expected_blob", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "expected_proofs", + "got": "expected_proofs", + "got_raw": "expected_proofs", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "HexBytes(go_tx_bytes)", + "got": "go_tx_bytes", + "got_raw": "go_tx_bytes", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.6875 + }, + { + "repo": "ethereum/eth-account", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "model_dump", + "got": "json_model_dump", + "got_raw": "json_model_dump", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8 + }, + { + "repo": "ethereum/eth-account", + "expected": "key", + "got": "key", + "got_raw": "key", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "a2.address", + "got": "a2.address", + "got_raw": "a2.address", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "expected_address", + "got": "expected_address", + "got_raw": "expected_address", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "ValidationError, match=\"Path is not valid.*\")", + "got": "ValidationError)", + "got_raw": "ValidationError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5245901639344263 + }, + { + "repo": "ethereum/eth-account", + "expected": "ValidationError, match=\"Path.*is not valid.*\")", + "got": "ValidationError)", + "got_raw": "ValidationError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5161290322580645 + }, + { + "repo": "ethereum/eth-account", + "expected": "ValidationError, match=\"Language not detected .*\")", + "got": "ValidationError)", + "got_raw": "ValidationError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.48484848484848486 + }, + { + "repo": "ethereum/eth-account", + "expected": "ValidationError, match=\"Invalid choice for number of words.*\")", + "got": "ValidationError)", + "got_raw": "ValidationError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.41025641025641024 + }, + { + "repo": "ethereum/eth-account", + "expected": "ValidationError, match=\".* not a valid BIP39 mnemonic phrase!\")", + "got": "ValidationError)", + "got_raw": "ValidationError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.4050632911392405 + }, + { + "repo": "ethereum/eth-account", + "expected": "ValidationError, match=\"Invalid language choice 'pig_latin', must be one of.*\")", + "got": "ValidationError(\"Unknown language: 'fr'\")", + "got_raw": "ValidationError(\"Unknown language: 'fr'\")", + "exact_match": false, + "code_bleu": 0.26509024292235017, + "edit_similarity": 0.5166666666666667 + }, + { + "repo": "ethereum/eth-account", + "expected": "ValidationError)", + "got": "ValidationError)", + "got_raw": "ValidationError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "expected_signable", + "got": "expected_signable", + "got_raw": "expected_signable", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "64", + "got": "32", + "got_raw": "32", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "lang", + "got": "lang", + "got_raw": "lang", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "word", + "got": "word", + "got_raw": "word", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "expected_seed", + "got": "expected_seed", + "got_raw": "expected_seed", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "Language(lang)", + "got": "lang", + "got_raw": "lang", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "ethereum/eth-account", + "expected": "ValidationError)", + "got": "ValidationError)", + "got_raw": "ValidationError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "m.expand_word(\"\")", + "got": "m.expand_word(\"\")", + "got_raw": "m.expand_word(\"\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "expected_mnemonic", + "got": "expected_mnemonic", + "got_raw": "expected_mnemonic", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "m.expand_word(\" \")", + "got": "m.expand_word(\" \")", + "got_raw": "m.expand_word(\" \")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "m.expand_word(\"acb\")", + "got": "m.expand_word(\"acb\")", + "got_raw": "m.expand_word(\"acb\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "m.expand_word(\"acc\")", + "got": "m.expand_word(\"acc\")", + "got_raw": "m.expand_word(\"acc\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "m.expand_word(\"act\")", + "got": "m.expand_word(\"act\")", + "got_raw": "m.expand_word(\"act\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "Language.CHINESE_SIMPLIFIED", + "got": "Language.SIMPLIFIED_CHINESE", + "got_raw": "Language.SIMPLIFIED_CHINESE", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7037037037037037 + }, + { + "repo": "ethereum/eth-account", + "expected": "Mnemonic.detect_language(word)", + "got": "Language.from_word(word)", + "got_raw": "Language.from_word(word)", + "exact_match": false, + "code_bleu": 0.4347208719449915, + "edit_similarity": 0.48148148148148145 + }, + { + "repo": "ethereum/eth-account", + "expected": "unicode_decompose_string( expected_mnemonic )", + "got": "expected_mnemonic", + "got_raw": "expected_mnemonic", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.5483870967741935 + }, + { + "repo": "ethereum/eth-account", + "expected": "new_local_account.sign_typed_data(full_message=message)", + "got": "new_local_account.sign_message(message)", + "got_raw": "new_local_account.sign_message(message)", + "exact_match": false, + "code_bleu": 0.2869522062208836, + "edit_similarity": 0.8085106382978723 + }, + { + "repo": "ethereum/eth-account", + "expected": "tuple(RLP_STRUCTURED_ACCESS_LIST)", + "got": "RLP_STRUCTURED_ACCESS_LIST", + "got_raw": "RLP_STRUCTURED_ACCESS_LIST", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.8813559322033898 + }, + { + "repo": "ethereum/eth-account", + "expected": "tuple(RPC_STRUCTURED_ACCESS_LIST)", + "got": "RPC_STRUCTURED_ACCESS_LIST", + "got_raw": "RPC_STRUCTURED_ACCESS_LIST", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.8813559322033898 + }, + { + "repo": "ethereum/eth-account", + "expected": "{\"testPydanticModel\": PYDANTIC_TEST_CLASS_JSON}", + "got": "{\"testPydanticModel\": {\"testPydanticModel\": {\"test\": \"test\"}}}", + "got_raw": "{\"testPydanticModel\": {\"testPydanticModel\": {\"test\": \"test\"}}}", + "exact_match": false, + "code_bleu": 0.18798317647335086, + "edit_similarity": 0.44036697247706424 + }, + { + "repo": "ethereum/eth-account", + "expected": "ValueError, match=\"provided object not formatted as rlp-structured access list\")", + "got": "pytest.raises(ValueError)", + "got_raw": " pytest.raises(ValueError)", + "exact_match": false, + "code_bleu": 0.24446151121745047, + "edit_similarity": 0.20952380952380953 + }, + { + "repo": "ethereum/eth-account", + "expected": "ValueError, match=\"provided object not formatted as JSON-RPC-structured access list\")", + "got": "pytest.raises(ValueError)", + "got_raw": " pytest.raises(ValueError)", + "exact_match": false, + "code_bleu": 0.24446151121745047, + "edit_similarity": 0.2 + }, + { + "repo": "ethereum/eth-account", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "expected.as_dict()", + "got": "expected.as_dict()", + "got_raw": "expected.as_dict()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "HexBytes(raw_transaction)", + "got": "HexBytes(raw_transaction)", + "got_raw": "HexBytes(raw_transaction)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "str(excinfo.value)", + "got": "excinfo.value.args[0]", + "got_raw": "excinfo.value.args[0]", + "exact_match": false, + "code_bleu": 0.2984745896009823, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "ethereum/eth-account", + "expected": "(TypeError, ABITypeError))", + "got": "ABITypeError)", + "got_raw": "ABITypeError)", + "exact_match": false, + "code_bleu": 0.23570226039551584, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "ethereum/eth-account", + "expected": "ValueError)", + "got": "ValidationError)", + "got_raw": "ValidationError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "ethereum/eth-account", + "expected": "ValidationError)", + "got": "ValidationError)", + "got_raw": "ValidationError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "len(set(string_test_cases))", + "got": "len(set(string_test_cases))", + "got_raw": "len(set(string_test_cases))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "encode_typed_data( *convert_to_3_arg(all_valid[message]) )", + "got": "all_valid[message]", + "got_raw": "all_valid[message]", + "exact_match": false, + "code_bleu": 0.36363636363636365, + "edit_similarity": 0.47368421052631576 + }, + { + "repo": "ethereum/eth-account", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "hash_type_expected", + "got": "hash_type_expected", + "got_raw": "hash_type_expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "encode_type_expected", + "got": "encode_type_expected", + "got_raw": "encode_type_expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "encode_data_expected", + "got": "encode_data_expected", + "got_raw": "encode_data_expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "hash_struct_expected", + "got": "hash_struct_expected", + "got_raw": "hash_struct_expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "ethers_sig", + "got": "ethers_sig", + "got_raw": "ethers_sig", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "py_three_arg", + "got": "metamask_sig", + "got_raw": "metamask_sig", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.25 + }, + { + "repo": "ethereum/eth-account", + "expected": "metamask_sig", + "got": "metamask_sig", + "got_raw": "metamask_sig", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ethereum/eth-account", + "expected": "ethers_address", + "got": "ethers_address", + "got_raw": "ethers_address", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "word", + "got": "word", + "got_raw": "word", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "wp_word", + "got": "wp_word", + "got_raw": "wp_word", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "X.shape", + "got": "(X_shape[0]", + "got_raw": "(X_shape[0], X_shape[1])", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "len(nested_align)", + "got": "align.lengths.shape[0]", + "got_raw": "align.lengths.shape[0]", + "exact_match": false, + "code_bleu": 0.13485111859503685, + "edit_similarity": 0.2564102564102564 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "align.data.shape[0]", + "got": "X_shape[0]", + "got_raw": "X_shape[0]", + "exact_match": false, + "code_bleu": 0.334370152488211, + "edit_similarity": 0.5517241379310345 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "align.lengths.shape[0]", + "got": "len(align)", + "got_raw": "len(align)", + "exact_match": false, + "code_bleu": 0.17965205598154213, + "edit_similarity": 0.3125 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "tagger_trf", + "got": "tagger_trf", + "got_raw": "tagger_trf", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "\"custom_transformer\"", + "got": "\"custom_transformer\"", + "got_raw": "\"custom_transformer\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "wp_dict[key])", + "got": "wp_dict[key])", + "got_raw": "wp_dict[key])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "aligned_wps", + "got": "aligned_wps", + "got_raw": "aligned_wps", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "1", + "got": "1000", + "got_raw": "1000", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "0", + "got": "(0", + "got_raw": "(0, 0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "()", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "0.0", + "got": "0.0", + "got_raw": "0.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "[\"ner\"]", + "got": "[\"ner\"]", + "got_raw": "[\"ner\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "normal_tags", + "got": "normal_tags", + "got_raw": "normal_tags", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "512", + "got": "512", + "got_raw": "512", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "499", + "got": "499", + "got_raw": "499", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "[\"V\"]", + "got": "[\"V\"]", + "got_raw": "[\"V\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "[\"tagger\"]", + "got": "[\"transformer\"]", + "got_raw": "[\"transformer\"]", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.64 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "6", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "\".\"", + "got": "\".\"", + "got_raw": "\".\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "\"One.\"", + "got": "\"one.\"", + "got_raw": "\"one.\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "\"One more.\"", + "got": "\"One more.\"", + "got_raw": "\"One more.\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "\"And one more.\"", + "got": "\"And\"", + "got_raw": "\"And\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "\"Three sentences in total.\"", + "got": "\"Three sentences in total.\"", + "got_raw": "\"Three sentences in total.\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "[\"textcat\"]", + "got": "[\"textcat\"]", + "got_raw": "[\"textcat\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "[\"tagger\"]", + "got": "[\"tagger\"]", + "got_raw": "[\"tagger\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "AssertionError)", + "got": "AssertionError)", + "got_raw": "AssertionError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "max_length", + "got": "max_length", + "got_raw": "max_length", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "align.lengths.shape[0]", + "got": "align.lengths.shape[0]", + "got_raw": "align.lengths.shape[0]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "wordpieces.strings[i][:max_length]", + "got": "wordpieces.strings[i]", + "got_raw": "wordpieces.strings[i]", + "exact_match": false, + "code_bleu": 0.6, + "edit_similarity": 0.7636363636363637 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "ops.to_numpy(t2))", + "got": "ytensor_equal(t1", + "got_raw": "ytensor_equal(t1, t2)", + "exact_match": false, + "code_bleu": 0.19362900077353815, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "1e-5", + "got": "0.0001", + "got_raw": "0.0001", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.2 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "cats2", + "got": "cats2", + "got_raw": "cats2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "len(cats2)", + "got": "len(cats2)", + "got_raw": "len(cats2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "[\"transformer\", \"textcat\"]", + "got": "[\"transformer\", \"textcat\"]", + "got_raw": "[\"transformer\", \"textcat\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-transformers", + "expected": "[doc[0].tag_ for doc in docs2[:-1]]", + "got": "[\"N\"", + "got_raw": "[\"N\", \"V\", \"J\", \"N\"]", + "exact_match": false, + "code_bleu": 0.06320999598962011, + "edit_similarity": 0.05128205128205128 + }, + { + "repo": "gradio-app/trackio", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "0.1", + "got": "0.1", + "got_raw": "0.1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "0.2", + "got": "0.2", + "got_raw": "0.2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "8", + "got": "lora_config.r", + "got_raw": "lora_config.r", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "16", + "got": "16", + "got_raw": "16", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "42", + "got": "42", + "got_raw": "42", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "75", + "got": "75", + "got_raw": "75", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "65", + "got": "65", + "got_raw": "65", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "0.1", + "got": "0.1", + "got_raw": "0.1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "0.2", + "got": "0.2", + "got_raw": "0.2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "i", + "got": "i", + "got_raw": "i", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "0.2", + "got": "0.1", + "got_raw": "0.1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "gradio-app/trackio", + "expected": "2", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "65", + "got": "25", + "got_raw": "25", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "gradio-app/trackio", + "expected": "64", + "got": "25", + "got_raw": "25", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "31", + "got": "31", + "got_raw": "31", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "30", + "got": "30", + "got_raw": "30", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "0.1", + "got": "0.1", + "got_raw": "0.1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "0.2", + "got": "0.2", + "got_raw": "0.2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "4", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "52", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "72", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "82", + "got": "82", + "got_raw": "82", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "run_names", + "got": "run_names", + "got_raw": "run_names", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "summary[\"metrics\"]", + "got": "summary", + "got_raw": "summary", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.56 + }, + { + "repo": "gradio-app/trackio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "num_logs", + "got": "num_logs", + "got_raw": "num_logs", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "num_threads", + "got": "num_threads", + "got_raw": "num_threads", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "num_logs - 1", + "got": "num_logs - 1", + "got_raw": "num_logs - 1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "logs_per_thread", + "got": "logs_per_thread", + "got_raw": "logs_per_thread", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "num_threads * logs_per_thread", + "got": "num_threads * logs_per_thread", + "got_raw": "num_threads * logs_per_thread", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "max_abs", + "got": "max_abs", + "got_raw": "max_abs", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "channels", + "got": "channels", + "got_raw": "channels", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "SAMPLE_RATE", + "got": "SAMPLE_RATE", + "got_raw": "SAMPLE_RATE", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "RuntimeError, match=\"Call trackio.init\\\\(\\\\)\")", + "got": "RuntimeError)", + "got_raw": "RuntimeError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.4406779661016949 + }, + { + "repo": "gradio-app/trackio", + "expected": "ImportError, match=\"nvidia-ml-py is required\")", + "got": "ImportError)", + "got_raw": "ImportError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.41379310344827586 + }, + { + "repo": "gradio-app/trackio", + "expected": "65", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "64", + "got": "12", + "got_raw": "12", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "31", + "got": "len(np_hist)", + "got_raw": "len(np_hist)", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "30", + "got": "30", + "got_raw": "30", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "11", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "gradio-app/trackio", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "len(data)", + "got": "30", + "got_raw": "30", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "hist_dict", + "got": "hist_dict", + "got_raw": "hist_dict", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "\"trackio.histogram\"", + "got": "\"histogram\"", + "got_raw": "\"histogram\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7333333333333333 + }, + { + "repo": "gradio-app/trackio", + "expected": "ValueError, match=\"Must provide either sequence or np_histogram\")", + "got": "pytest.raises(TypeError)", + "got_raw": "pytest.raises(TypeError)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.1797752808988764 + }, + { + "repo": "gradio-app/trackio", + "expected": "ValueError, match=\"Cannot provide both sequence and np_histogram\")", + "got": "pytest.raises(TypeError)", + "got_raw": "pytest.raises(TypeError)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.17777777777777778 + }, + { + "repo": "gradio-app/trackio", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "ValueError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "gradio-app/trackio", + "expected": "\"test_caption\"", + "got": "\"test_caption\"", + "got_raw": "\"test_caption\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "TrackioImage.TYPE", + "got": "\"image\"", + "got_raw": "\"image\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.4166666666666667 + }, + { + "repo": "gradio-app/trackio", + "expected": "TrackioVideo.TYPE", + "got": "\"video\"", + "got_raw": "\"video\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "gradio-app/trackio", + "expected": "TrackioAudio.TYPE", + "got": "\"audio\"", + "got_raw": "\"audio\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.4166666666666667 + }, + { + "repo": "gradio-app/trackio", + "expected": "str(image._get_relative_file_path())", + "got": "str(image._file_path)", + "got_raw": "str(image._file_path)", + "exact_match": false, + "code_bleu": 0.48259416312057124, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "gradio-app/trackio", + "expected": "str(video._get_relative_file_path())", + "got": "str(video._file_path)", + "got_raw": "str(video._file_path)", + "exact_match": false, + "code_bleu": 0.48259416312057124, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "gradio-app/trackio", + "expected": "str(audio._get_relative_file_path())", + "got": "str(Path(temp_dir)", + "got_raw": "str(Path(temp_dir) / \"test_run\" / \"test_caption.wav\")", + "exact_match": false, + "code_bleu": 0.24773732385103553, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "gradio-app/trackio", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "3", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "4", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "0.01", + "got": "0.01", + "got_raw": "0.01", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "\"proj\"", + "got": "\"proj\"", + "got_raw": "\"proj\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "\"run1\"", + "got": "\"run1\"", + "got_raw": "\"run1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "metrics", + "got": "metrics", + "got_raw": "metrics", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "\"new-run\"", + "got": "\"new-run\"", + "got_raw": "\"new-run\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "\"testuser\"", + "got": "\"testuser\"", + "got_raw": "\"testuser\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "{}", + "got": "[(\"test_run\"", + "got_raw": "[(\"test_run\", \"test\")]", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "0.9", + "got": "0.9", + "got_raw": "0.9", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "1", + "got": "5", + "got_raw": "5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "42", + "got": "42", + "got_raw": "42", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "84", + "got": "84", + "got_raw": "84", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "1.5", + "got": "1.5", + "got_raw": "1.5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "(0, 5)", + "got": "(None, 5)", + "got_raw": "(None, 5)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "gradio-app/trackio", + "expected": "(2, 0)", + "got": "(2, 10)", + "got_raw": "(2, 10)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "gradio-app/trackio", + "expected": "len(df)", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "\"Unknown\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "gradio-app/trackio", + "expected": "test_path", + "got": "test_path", + "got_raw": "test_path", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "\"Just now\"", + "got": "\"30 seconds ago\"", + "got_raw": "\"30 seconds ago\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "gradio-app/trackio", + "expected": "\"1 hour ago\"", + "got": "\"1 hour ago\"", + "got_raw": "\"1 hour ago\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "\"2 days ago\"", + "got": "\"2 days ago\"", + "got_raw": "\"2 days ago\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "len(set(names))", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.07142857142857142, + "edit_similarity": 0.0 + }, + { + "repo": "gradio-app/trackio", + "expected": "\"/data/trackio\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "5", + "got": "\"EMPIRE\"", + "got_raw": "\"EMPIRE\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "1", + "got": "\"field\"", + "got_raw": "\"field\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "0", + "got": "\"l1\"", + "got_raw": "\"l1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "e", + "got": "e", + "got_raw": "e", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "k", + "got": "k", + "got_raw": "k", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "r", + "got": "r", + "got_raw": "r", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "s", + "got": "r", + "got_raw": "r", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "m", + "got": "m", + "got_raw": "m", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "v", + "got": "v", + "got_raw": "v", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "s", + "got": "obj1", + "got_raw": "obj1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "e2", + "got": "e2", + "got_raw": "e2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "e1", + "got": "e1", + "got_raw": "e1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "sdl", + "got": "\"schema", + "got_raw": "\"schema = GraphQLSchema(\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.2 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "out", + "got": "out", + "got_raw": "out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "loc", + "got": "loc", + "got_raw": "loc", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "res", + "got": "res", + "got_raw": "res", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "ast", + "got": "ast", + "got_raw": "ast", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "ast", + "got": "ast", + "got_raw": "ast", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "key", + "got": "key", + "got_raw": "key", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "ast", + "got": "ast", + "got_raw": "ast", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "str", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "obj", + "got": "obj", + "got_raw": "obj", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "res", + "got": "{", + "got_raw": "{ \"kind\": \"name\", \"value\": \"test\", \"locations\": [ { \"kind\": \"location\", \"name\": \"kind\", \"value\": \"name\" }, { \"kind\": \"location\", \"name\": \"value\", \"value\": \"test\" } ] }", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "ast", + "got": "ast", + "got_raw": "ast", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "sdl", + "got": "sdl", + "got_raw": "sdl", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "sdl", + "got": "sdl", + "got_raw": "sdl", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "type", + "got": "type(Exception)", + "got_raw": "type(Exception)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "args", + "got": "tuple(args.values())", + "got_raw": "tuple(args.values())", + "exact_match": false, + "code_bleu": 0.16036590969929357, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[]", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "()", + "got": "()", + "got_raw": "()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "{}", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "()", + "got": "()", + "got_raw": "()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "()", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "()", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[4]", + "got": "[0", + "got_raw": "[0, 1]", + "exact_match": false, + "code_bleu": 0.35818997727451396, + "edit_similarity": 0.4 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[0]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.39640237166757364, + "edit_similarity": 0.8 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[6]", + "got": "[6]", + "got_raw": "[6]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[2]", + "got": "(SourceLocation(1", + "got_raw": "(SourceLocation(1, 3),)", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[0]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.39640237166757364, + "edit_similarity": 0.8 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "{})", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.35818997727451396, + "edit_similarity": 0.4 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[])", + "got": "[", + "got_raw": "[ \"\"\"", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[42]", + "got": "[42]", + "got_raw": "[42]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[\"\"]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.39640237166757364, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "(2, 3)", + "got": "SourceLocation(2, 3)", + "got_raw": "SourceLocation(2, 3)", + "exact_match": false, + "code_bleu": 0.8091067115702212, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "(1, 1)", + "got": "SourceLocation(0, 0)", + "got_raw": "SourceLocation(0, 0)", + "exact_match": false, + "code_bleu": 0.2626909894424158, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "(2, 6)", + "got": "SourceLocation(1, 1)", + "got_raw": "SourceLocation(1, 1)", + "exact_match": false, + "code_bleu": 0.2626909894424158, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[[42]]", + "got": "[42]", + "got_raw": "[42]", + "exact_match": false, + "code_bleu": 0.5045378491522287, + "edit_similarity": 0.8 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[True]", + "got": "[True]", + "got_raw": "[True]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "(1, 2))", + "got": "(0, 1))", + "got_raw": "(0, 1))", + "exact_match": false, + "code_bleu": 0.345720784641941, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "(1, 4))", + "got": "(1, 3))", + "got_raw": "(1, 3))", + "exact_match": false, + "code_bleu": 0.488923022434901, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "(1, 5))", + "got": "(1, 5))", + "got_raw": "(1, 5))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "(1, 1))", + "got": "'\"no end quote\")\"\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")", + "got_raw": "'\"no end quote\")\"\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")\")", + "exact_match": false, + "code_bleu": 0.010404153511161668, + "edit_similarity": 0.022388059701492536 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "(1, 7))", + "got": "(1, 1))", + "got_raw": "(1, 1))", + "exact_match": false, + "code_bleu": 0.488923022434901, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[(1, 3)]", + "got": "[SourceLocation(1, 3)]", + "got_raw": "[SourceLocation(1, 3)]", + "exact_match": false, + "code_bleu": 0.7506238537503395, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[2, 4, 6]", + "got": "[2, 4, 6]", + "got_raw": "[2, 4, 6]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[0, 1, 2]", + "got": "[0, 1, 2, 3]", + "got_raw": "[0, 1, 2, 3]", + "exact_match": false, + "code_bleu": 0.6606328636027614, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[2, 1, 0]", + "got": "[0, 1, 2, 3]", + "got_raw": "[0, 1, 2, 3]", + "exact_match": false, + "code_bleu": 0.3356891925037239, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[1, 2, 3]", + "got": "[1, 2, 3]", + "got_raw": "[1, 2, 3]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[\"\", \"a\"]", + "got": "[\"a\"]", + "got_raw": "[\"a\"]", + "exact_match": false, + "code_bleu": 0.38336586254776345, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "({}, None)", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.23570226039551584, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "({}, None)", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.23570226039551584, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[0, \"one\"]", + "got": "[\"one\"]", + "got_raw": "[\"one\"]", + "exact_match": false, + "code_bleu": 0.38336586254776345, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "{\"foo\": 5}", + "got": "{\"foo\": 5}", + "got_raw": "{\"foo\": 5}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "{\"foo\": 7}", + "got": "{\"a\": 7}", + "got_raw": "{\"a\": 7}", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "(data, None)", + "got": "(data, errors)", + "got_raw": "(data, errors)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6923076923076923 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[foo_object]", + "got": "[foo_object]", + "got_raw": "[foo_object]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "(data, None)", + "got": "(data, errors)", + "got_raw": "(data, errors)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6923076923076923 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "(data, None)", + "got": "ExecutionResult(data={\"null\": None})", + "got_raw": "ExecutionResult(data={\"null\": None})", + "exact_match": false, + "code_bleu": 0.18850319022747347, + "edit_similarity": 0.4583333333333333 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "{\"foo\": 123}", + "got": "{\"foo\": 123}", + "got_raw": "{\"foo\": 123}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "{\"id\": \"foo\"}", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[\"listField\"]", + "got": "[\"apple\"", + "got_raw": "[\"apple\", \"banana\", \"coconut\"]", + "exact_match": false, + "code_bleu": 0.35818997727451396, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[0, \"one\", 2]", + "got": "[0, \"one\", 2]", + "got_raw": "[0, \"one\", 2]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "{\"foo\": None}", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "{\"data\": data}", + "got": "data", + "got_raw": "data", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "(data, errors)", + "got": "(data, errors)", + "got_raw": "(data, errors)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "(None, errors)", + "got": "(data, errors)", + "got_raw": "(data, errors)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "[data, errors]", + "got": "[data]", + "got_raw": "[data]", + "exact_match": false, + "code_bleu": 0.38336586254776345, + "edit_similarity": 0.6 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "(None, errors)", + "got": "({\"listField\": None}, errors)", + "got_raw": "({\"listField\": None}, errors)", + "exact_match": false, + "code_bleu": 0.2907153684841096, + "edit_similarity": 0.6511627906976745 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "{\"bar\": \"rab\"}", + "got": "{\"bar\": \"oof\"", + "got_raw": "{\"bar\": \"oof\", \"bar\": \"bar\"}", + "exact_match": false, + "code_bleu": 0.5349922439811376, + "edit_similarity": 0.7407407407407407 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "{\"bar\": \"foo\"}", + "got": "{\"bar\": \"oof\"}", + "got_raw": "{\"bar\": \"oof\"}", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.9285714285714286 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "{\"hasNext\": True}", + "got": "{\"hasNext\": True}", + "got_raw": "{\"hasNext\": True}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "({\"a\": {}}, None)", + "got": "{\"a\": {\"b\": \"c\"}}", + "got_raw": "{\"a\": {\"b\": \"c\"}}", + "exact_match": false, + "code_bleu": 0.40662090162443015, + "edit_similarity": 0.5294117647058824 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "{\"test\": \"test\"})", + "got": "{})", + "got_raw": "{})", + "exact_match": false, + "code_bleu": 0.3194715521231362, + "edit_similarity": 0.3 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "{\"test\": \"test\"})", + "got": "{})", + "got_raw": "{})", + "exact_match": false, + "code_bleu": 0.3194715521231362, + "edit_similarity": 0.3 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "{\"hasNext\": False}", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.2 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "({\"a\": \"a\"}, None)", + "got": "ExecutionResult({\"a\": None, \"b\": None})", + "got_raw": "ExecutionResult({\"a\": None, \"b\": None})", + "exact_match": false, + "code_bleu": 0.3349531889697646, + "edit_similarity": 0.49122807017543857 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "({\"a\": \"b\"}, None)", + "got": "{\"a\": \"b\"}", + "got_raw": "{\"a\": \"b\"}", + "exact_match": false, + "code_bleu": 0.5555555555555556, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "({\"c\": \"d\"}, None)", + "got": "{ \"M\": {\"c\": \"d\"} }", + "got_raw": "{ \"M\": {\"c\": \"d\"} }", + "exact_match": false, + "code_bleu": 0.5169731539571706, + "edit_similarity": 0.5945945945945946 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "key(\"a\")", + "got": "key(\"", + "got_raw": "key(\" \")", + "exact_match": false, + "code_bleu": 0.4459526681260204, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "key(\"1\")", + "got": "key(\"", + "got_raw": "key(\" \")", + "exact_match": false, + "code_bleu": 0.4459526681260204, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "key(\"2\")", + "got": "key(\"2\")", + "got_raw": "key(\"2\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "key(\"AA\")", + "got": "key(\"B\")", + "got_raw": "key(\"B\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "key(\"11\")", + "got": "key(\"1\")", + "got_raw": "key(\"1\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.9411764705882353 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "key(\"00\")", + "got": "key(\"1\")", + "got_raw": "key(\"1\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "key(\"A1A\")", + "got": "key(\"AA\")", + "got_raw": "key(\"AA\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "key(\"200\")", + "got": "key(\"012\")", + "got_raw": "key(\"012\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "key(\"a9a\")", + "got": "key(\"a1a\")", + "got_raw": "key(\"a1a\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.9 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "hash(None)", + "got": "hash(1)", + "got_raw": "hash(1)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "hash(True)", + "got": "hash(Undefined)", + "got_raw": "hash(Undefined)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.56 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "key(\"a09a\")", + "got": "key(\"a01a\")", + "got_raw": "key(\"a01a\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "key(\"a0a9\")", + "got": "key(\"a0a2\")", + "got_raw": "key(\"a0a2\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "hash(False)", + "got": "hash(1)", + "got_raw": "hash(1)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "test_value())", + "got": "_values(test_value())", + "got_raw": "_values(test_value())", + "exact_match": false, + "code_bleu": 0.6147881529512643, + "edit_similarity": 0.7647058823529411 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"\"", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"n\"", + "got": "\"n\"", + "got_raw": "\"n\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"A\"", + "got": "\"A\"", + "got_raw": "\"A\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"0\"", + "got": "0.0", + "got_raw": "0.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"\")", + "got": "\"\")", + "got_raw": "\"\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"l2\"", + "got": "\"l1\"", + "got_raw": "\"l1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"[]\"", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"()\"", + "got": "()", + "got_raw": "()", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"{}\"", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"baz\"", + "got": "\"custom\"", + "got_raw": "\"custom\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "'\" \"'", + "got": "\" \"", + "got_raw": "\" \"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"bar\"", + "got": "\"bar\"", + "got_raw": "\"bar\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"Foo\"", + "got": "\"Foo\"", + "got_raw": "\"Foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"0.0\"", + "got": "\"0.0\"", + "got_raw": "\"0.0\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"314\"", + "got": "\"314\"", + "got_raw": "\"314\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"nan\"", + "got": "\"NaN\"", + "got_raw": "\"NaN\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"inf\"", + "got": "\"inf\"", + "got_raw": "\"inf\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"Int\"", + "got": "GraphQLInt", + "got_raw": "GraphQLInt", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"two\"", + "got": "\"two\"", + "got_raw": "\"two\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"[1]\"", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"Foo\"", + "got": "\"Foo\"", + "got_raw": "\"Foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"loc\"", + "got": "\"definitions\"", + "got_raw": "\"definitions\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"1.5\"", + "got": "\"1.5\"", + "got_raw": "\"1.5\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"Cat\"", + "got": "\"Cat\"", + "got_raw": "\"Cat\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"lazy\"", + "got": "\"Lazy\"", + "got_raw": "\"Lazy\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"ouch\"", + "got": "\"ouch\"", + "got_raw": "\"ouch\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"None\"", + "got": "\"None\"", + "got_raw": "\"None\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"True\"", + "got": "\"true\"", + "got_raw": "\"true\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\".one\"", + "got": "\"one\"", + "got_raw": "\"one\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"test\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"@foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"kind\"", + "got": "\"kind\"", + "got_raw": "\"kind\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"alpha\"", + "got": "\"alpha\"", + "got_raw": "\"alpha\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"1.2.3\"", + "got": "\"1.2.3\"", + "got_raw": "\"1.2.3\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"Query\"", + "got": "\"Lazy\"", + "got_raw": "\"Lazy\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"First\"", + "got": "\"First\"", + "got_raw": "\"First\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"query\"", + "got": "{\"type\":", + "got_raw": "{\"type\": \"query\"}", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.4 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "'\"foo\"'", + "got": "'\"foo\"'", + "got_raw": "'\"foo\"'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"A or B\"", + "got": "\"A or B\"", + "got_raw": "\"A or B\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"Second\"", + "got": "\"Second\"", + "got_raw": "\"Second\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"1.2.3a4\"", + "got": "\"1.2.3a4\"", + "got_raw": "\"1.2.3a4\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"Goodbye\"", + "got": "\"Goodbye\"", + "got_raw": "\"Goodbye\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"A and B\"", + "got": "[\"A\"", + "got_raw": "[\"A\", \"B\"]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"1.2.3rc4\"", + "got": "\"1.2.3c4\"", + "got_raw": "\"1.2.3c4\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"resolved\"", + "got": "\"resolved\"", + "got_raw": "\"resolved\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "'\"\\u21bb\"'", + "got": "\"\\u21bb\"", + "got_raw": "\"\\u21bb\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"not lazy\"", + "got": "\"not lazy\"", + "got_raw": "\"not lazy\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "'\"\\\\u0000\"'", + "got": "\"\\\\u0000\"", + "got_raw": "\"\\\\u0000\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"SnakeCase\"", + "got": "\"SnakeCase\"", + "got_raw": "\"SnakeCase\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"CamelCase\"", + "got": "\"CamelCase\"", + "got_raw": "\"CamelCase\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"snakeCase\"", + "got": "\"SnakeCase\"", + "got_raw": "\"SnakeCase\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"lazyField\"", + "got": "\"lazyField\"", + "got_raw": "\"lazyField\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"Undefined\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9166666666666666 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\">--()-->\")", + "got": "\"InvalidName\")", + "got_raw": "\"InvalidName\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.24 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"type Bar\")", + "got": "schema)", + "got_raw": "schema)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"type Foo\")", + "got": "schema)", + "got_raw": "schema)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "\"Some error\"", + "got": "\"ValueError(Some error)\"", + "got_raw": "\"ValueError(Some error)\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "True", + "got": "payload1", + "got_raw": "payload1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "1 + 2j", + "got": "complex(1", + "got_raw": "complex(1, 2)", + "exact_match": false, + "code_bleu": 0.3388507513536918, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "*items)", + "got": "*items)", + "got_raw": "*items)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphql-core", + "expected": "values[:-1]", + "got": "values", + "got_raw": "values", + "exact_match": false, + "code_bleu": 0.09910059291689341, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "grll/mcpadapt", + "expected": "5", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "2.5", + "got": "timedelta(seconds=2.5)", + "got_raw": "timedelta(seconds=2.5)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.24 + }, + { + "repo": "grll/mcpadapt", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "\"echo_tool\"", + "got": "\"echo_tool\"", + "got_raw": "\"echo_tool\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "\"echo_tool\"", + "got": "\"echo_tool\"", + "got_raw": "\"echo_tool\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "\"Echo: hello\"", + "got": "\"Echo: Hello\"", + "got_raw": "\"Echo: Hello\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "grll/mcpadapt", + "expected": "\"Echo: empty\"", + "got": "tool_functions[\"echo_tool_default_value\"]({})", + "got_raw": "tool_functions[\"echo_tool_default_value\"]({})", + "exact_match": false, + "code_bleu": 0.13485111859503685, + "edit_similarity": 0.20689655172413793 + }, + { + "repo": "grll/mcpadapt", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "\"def_\"", + "got": "\"def\"", + "got_raw": "\"def\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "grll/mcpadapt", + "expected": "result_int", + "got": "result_int", + "got_raw": "result_int", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "\"echo_tool\"", + "got": "\"echo_tool\"", + "got_raw": "\"echo_tool\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "\"Echo: hello\"", + "got": "\"Echo: hello\"", + "got_raw": "\"Echo: hello\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "result_string", + "got": "result_string", + "got_raw": "result_string", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "70", + "got": "70", + "got_raw": "70", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "\"def_\"", + "got": "\"def\"", + "got_raw": "\"def\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "grll/mcpadapt", + "expected": "\"sunny\"", + "got": "\"sunny\"", + "got_raw": "\"sunny\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "(256, 256)", + "got": "(128, 128)", + "got_raw": "(128, 128)", + "exact_match": false, + "code_bleu": 0.32466791547509893, + "edit_similarity": 0.6 + }, + { + "repo": "grll/mcpadapt", + "expected": "\"echo_tool\"", + "got": "\"echo_tool\"", + "got_raw": "\"echo_tool\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "\"test_image\"", + "got": "\"test_image\"", + "got_raw": "\"test_image\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "\"test_audio\"", + "got": "\"test_audio\"", + "got_raw": "\"test_audio\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "\"Echo: hello\"", + "got": "\"Echo: hello\"", + "got_raw": "\"Echo: hello\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "42", + "got": "42", + "got_raw": "42", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grll/mcpadapt", + "expected": "\"test\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "serialized", + "got": "serialized", + "got_raw": "serialized", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "expected_attrs", + "got": "expected_attrs", + "got_raw": "expected_attrs", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "'>'", + "got": "'>'", + "got_raw": "'>'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "'\\u03B2'", + "got": "b\"β\"", + "got_raw": "b\"β\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.3448275862068966 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "'β'", + "got": "doc", + "got_raw": "doc", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.058823529411764705 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "s", + "got": "s", + "got_raw": "s", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "6", + "got": "6", + "got_raw": "6", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "b\"\"", + "got": "b\"\"", + "got_raw": "b\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "\"'\"", + "got": "b\"'\"", + "got_raw": "b\"'\"", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "\"x\"", + "got": "\"xe\"", + "got_raw": "\"xe\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "\"a\"", + "got": "\"b\"", + "got_raw": "\"b\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "\"b\"", + "got": "\"b\"", + "got_raw": "\"b\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "\"c\"", + "got": "\"c\"", + "got_raw": "\"c\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "\"d\"", + "got": "\"d\"", + "got_raw": "\"d\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "num", + "got": "num", + "got_raw": "num", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "1", + "got": "len(attrs)", + "got_raw": "len(attrs)", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "out_name", + "got": "out_name", + "got_raw": "out_name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "out_value", + "got": "out_value", + "got_raw": "out_value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "len(attrs)", + "got": "len(attrs)", + "got_raw": "len(attrs)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "[('a', '1')]", + "got": "[(\"a\", \"1\")]", + "got_raw": "[(\"a\", \"1\")]", + "exact_match": false, + "code_bleu": 0.32172944208038085, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "tokenTypes['StartTag']", + "got": "tokenTypes['StartTag']", + "got_raw": "tokenTypes['StartTag']", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "handler.visited", + "got": "handler.events", + "got_raw": "handler.events", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6206896551724138 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "2", + "got": "len(attrs)", + "got_raw": "len(attrs)", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "\"EndTag\"", + "got": "'EndTag'", + "got_raw": "'EndTag'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "out_name", + "got": "(None", + "got_raw": "(None, out_name)", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "out_value", + "got": "out_value", + "got_raw": "out_value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "'StartTag'", + "got": "\"attribute\"", + "got_raw": "\"attribute\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "outputToken", + "got": "outputToken", + "got_raw": "outputToken", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "[((None, 'a'), '1'), ((None, 'refX'), '2'), ((None, 'b'), '3'), (('http://www.w3.org/XML/1998/namespace', 'lang'), '4'), ((None, 'c'), '5')]", + "got": "[(\"a\", \"1\"), (\"refx\", \"2\"), (\"b\", \"3\"), (\"xml:lang\", \"4\"), (\"c\", \"5\")]", + "got_raw": "[(\"a\", \"1\"), (\"refx\", \"2\"), (\"b\", \"3\"), (\"xml:lang\", \"4\"), (\"c\", \"5\")]", + "exact_match": false, + "code_bleu": 0.11761971672732877, + "edit_similarity": 0.4095238095238095 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "output", + "got": "output", + "got_raw": "output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "output", + "got": "output", + "got_raw": "output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "html5lib/html5lib-python", + "expected": "len(expectedErrors)", + "got": "len(expectedErrors)", + "got_raw": "len(expectedErrors)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "30", + "got": "30", + "got_raw": "30", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "70", + "got": "70", + "got_raw": "70", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "100", + "got": "100", + "got_raw": "100", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "['test']", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.39640237166757364, + "edit_similarity": 0.4 + }, + { + "repo": "insanum/gcalcli", + "expected": "SystemExit)", + "got": "argparse.ArgumentTypeError)", + "got_raw": "argparse.ArgumentTypeError)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.2631578947368421 + }, + { + "repo": "insanum/gcalcli", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "[minimal_event]", + "got": "[('minimal_event'", + "got_raw": "[('minimal_event', 'minimal_event_overlapping')]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.875 + }, + { + "repo": "insanum/gcalcli", + "expected": "0", + "got": "''", + "got_raw": "''", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "(5, 5)", + "got": "test_day_width", + "got_raw": "test_day_width", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "(7, 7)", + "got": "(4, 4)", + "got_raw": "(4, 4)", + "exact_match": false, + "code_bleu": 0.32466791547509893, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "insanum/gcalcli", + "expected": "(8, 6)", + "got": "(10, 10)", + "got_raw": "(10, 10)", + "exact_match": false, + "code_bleu": 0.32466791547509893, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "insanum/gcalcli", + "expected": "captured.out", + "got": "captured", + "got_raw": "captured", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.8 + }, + { + "repo": "insanum/gcalcli", + "expected": "\"1m\"", + "got": "\"00:01\"", + "got_raw": "\"00:01\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "insanum/gcalcli", + "expected": "\"1h2m\"", + "got": "\"1h2m\"", + "got_raw": "\"1h2m\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "\"Valid Text\"", + "got": "\"Valid Text\"", + "got_raw": "\"Valid Text\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "ValidationError)", + "got": "ValidationError)", + "got_raw": "ValidationError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "'Expected format: a duration (e.g. 1m, 1s, 1h3m)' '(Ctrl-C to exit)\\n'", + "got": "", + "got_raw": "", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "'msg'", + "got": "'msg'", + "got_raw": "'msg'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "ArgumentTypeError)", + "got": "ArgumentTypeError)", + "got_raw": "ArgumentTypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "'\\033[0;31mmsg\\033[0m'", + "got": "'\\\\033[31mmsg\\\\033[0m\\n'", + "got_raw": "'\\\\033[31mmsg\\\\033[0m\\n'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8695652173913043 + }, + { + "repo": "insanum/gcalcli", + "expected": "'\\033[31;1merror\\033[0m'", + "got": "'error\\n'", + "got_raw": "'error\\n'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.48484848484848486 + }, + { + "repo": "insanum/gcalcli", + "expected": "'\\033[0;33mdebug\\033[0m'", + "got": "''", + "got_raw": "''", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "insanum/gcalcli", + "expected": "'${color red}msg${color}'", + "got": "'${color red}msg${color default}\\n'", + "got_raw": "'${color red}msg${color default}\\n'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "insanum/gcalcli", + "expected": "'${color red}error${color}'", + "got": "'error\\n'", + "got_raw": "'error\\n'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3888888888888889 + }, + { + "repo": "insanum/gcalcli", + "expected": "'${color yellow}debug${color}'", + "got": "'conky: debug\\n'", + "got_raw": "'conky: debug\\n'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.43478260869565216 + }, + { + "repo": "insanum/gcalcli", + "expected": "cp.colors[color_name] + 'msg' + '\\033[0m'", + "got": "f'\\x1b[38;5;{color_name}mmsg\\x1b[0m\\n'", + "got_raw": "f'\\x1b[38;5;{color_name}mmsg\\x1b[0m\\n'", + "exact_match": false, + "code_bleu": 0.09036020036098448, + "edit_similarity": 0.4810126582278481 + }, + { + "repo": "insanum/gcalcli", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "364", + "got": "364", + "got_raw": "364", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "timedelta(hours=1)", + "got": "timedelta(hours=1)", + "got_raw": "timedelta(hours=1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "insanum/gcalcli", + "expected": "timedelta(minutes=1)", + "got": "timedelta(hours=1)", + "got_raw": "timedelta(hours=1)", + "exact_match": false, + "code_bleu": 0.488923022434901, + "edit_similarity": 0.7894736842105263 + }, + { + "repo": "insanum/gcalcli", + "expected": "\"Duration is invalid: junk\"", + "got": "'Invalid timedelta format: junk'", + "got_raw": "'Invalid timedelta format: junk'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4067796610169492 + }, + { + "repo": "insanum/gcalcli", + "expected": "utils.get_times_from_duration(begin_1970_midnight, duration=120)", + "got": "begin_1970_midnight", + "got_raw": "begin_1970_midnight", + "exact_match": false, + "code_bleu": 0.05946035575013606, + "edit_similarity": 0.4578313253012048 + }, + { + "repo": "insanum/gcalcli", + "expected": "utils.get_times_from_duration( begin_1970_midnight, duration=\"2h\")", + "got": "next_day)", + "got_raw": "next_day)", + "exact_match": false, + "code_bleu": 0.1074569931823542, + "edit_similarity": 0.16 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "5.0", + "got": "5.0", + "got_raw": "5.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "6.0", + "got": "6.0", + "got_raw": "6.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "10.0", + "got": "10.0", + "got_raw": "10.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0", + "got": "max_size", + "got_raw": "max_size", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "N", + "got": "N", + "got_raw": "N", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "[0]", + "got": "[0.0]", + "got_raw": "[0.0]", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.75 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "items", + "got": "items", + "got_raw": "items", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "flushes", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "max_size", + "got": "max_size", + "got_raw": "max_size", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "42", + "got": "42", + "got_raw": "42", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "ttl", + "got": "ttl", + "got_raw": "ttl", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "value1", + "got": "value1", + "got_raw": "value1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"value\"", + "got": "\"value\"", + "got_raw": "\"value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"value_1\"", + "got": "\"value_1\"", + "got_raw": "\"value_1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "await cache()", + "got": "\"cached_value\"", + "got_raw": "\"cached_value\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.37037037037037035 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "CustomException)", + "got": "CustomException)", + "got_raw": "CustomException)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "N", + "got": "N", + "got_raw": "N", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0", + "got": "N", + "got_raw": "N", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"sync\"", + "got": "\"test_value\"", + "got_raw": "\"test_value\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"async\"", + "got": "\"test_value\"", + "got_raw": "\"test_value\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3157894736842105 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"drain\"", + "got": "\"test_value\"", + "got_raw": "\"test_value\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3157894736842105 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"async_cm\"", + "got": "\"test_value\"", + "got_raw": "\"test_value\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "NotImplementedError)", + "got": "asyncio.CancelledError)", + "got_raw": "asyncio.CancelledError)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.4186046511627907 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "N", + "got": "N", + "got_raw": "N", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "tally.max_active", + "got": "tally.count", + "got_raw": "tally.count", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.5925925925925926 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "2 * concurrency_limit", + "got": "2 * concurrency_limit", + "got_raw": "2 * concurrency_limit", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "{}", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "payload", + "got": "payload", + "got_raw": "payload", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "str(exc)", + "got": "str(exc)", + "got_raw": "str(exc)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "Exception)", + "got": "asyncpg.exceptions.OperationalError)", + "got_raw": "asyncpg.exceptions.OperationalError)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.391304347826087 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "expected_policy", + "got": "expected_policy", + "got_raw": "expected_policy", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "expected_logged[\"pgqueuer\"]", + "got": "expected_logged[\"pgqueuer\"]", + "got_raw": "expected_logged[\"pgqueuer\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "expected_logged[\"pgqueuer_log\"]", + "got": "expected_logged[\"pgqueuer_log\"]", + "got_raw": "expected_logged[\"pgqueuer_log\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "expected_logged[\"pgqueuer_schedules\"]", + "got": "expected_logged[\"pgqueuer_schedules\"]", + "got_raw": "expected_logged[\"pgqueuer_schedules\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "expected_logged[\"pgqueuer_statistics\"]", + "got": "expected_logged[\"pgqueuer_statistics\"]", + "got_raw": "expected_logged[\"pgqueuer_statistics\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "ValueError, match=\"'nonexistent' is not a valid Durability\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.30985915492957744 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "1", + "got": "timedelta(seconds=timedelta(seconds=1).total_seconds())", + "got_raw": "timedelta(seconds=timedelta(seconds=1).total_seconds())", + "exact_match": false, + "code_bleu": 0.08225964699966554, + "edit_similarity": 0.03571428571428571 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "42", + "got": "42", + "got_raw": "42", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"/fake/dir\"", + "got": "\"/fake/dir\"", + "got_raw": "\"/fake/dir\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "mock_factory", + "got": "mock_factory", + "got_raw": "mock_factory", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"sync_cm_value\"", + "got": "\"sync_cm_value\"", + "got_raw": "\"sync_cm_value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"async_cm_value\"", + "got": "\"async_cm_value\"", + "got_raw": "\"async_cm_value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"awaitable_value\"", + "got": "\"awaitable_value\"", + "got_raw": "\"awaitable_value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "ImportError, match=\"No module named 'nonexistent_module'\")", + "got": "ImportError)", + "got_raw": "ImportError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.34285714285714286 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "AttributeError, match=\"Mock object has no attribute 'nonexistent_factory'\")", + "got": "AttributeError)", + "got_raw": "AttributeError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "{ \"object str can't be used in 'await' expression\", \"'str' object can't be awaited\", }", + "got": "\"Invalid input type\"", + "got_raw": "\"Invalid input type\"", + "exact_match": false, + "code_bleu": 0.08333333333333333, + "edit_similarity": 0.22641509433962265 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0.1", + "got": "1.0", + "got_raw": "1.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "3", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "t1", + "got": "t1", + "got_raw": "t1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "elapsed()", + "got": "0.0", + "got_raw": "0.0", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "ValueError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "timedelta(1)", + "got": "timedelta(1)", + "got_raw": "timedelta(1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "timedelta(2)", + "got": "timedelta(1)", + "got_raw": "timedelta(1)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.9166666666666666 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "timedelta(4)", + "got": "timedelta(4)", + "got_raw": "timedelta(4)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "timedelta(8)", + "got": "timedelta(8)", + "got_raw": "timedelta(8)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "n", + "got": "n", + "got_raw": "n", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "2", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "500", + "got": "10_000", + "got_raw": "10_000", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "5_000", + "got": "10_000", + "got_raw": "10_000", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "statuses", + "got": "statuses", + "got_raw": "statuses", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "sorted(order, reverse=True)", + "got": "[5, 10, 1]", + "got_raw": "[5, 10, 1]", + "exact_match": false, + "code_bleu": 0.162531757492419, + "edit_similarity": 0.10810810810810811 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"insert\"", + "got": "\"insert\"", + "got_raw": "\"insert\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"truncate\"", + "got": "\"truncate\"", + "got_raw": "\"truncate\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "event.root.id", + "got": "event.root.id", + "got_raw": "event.root.id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "NotImplementedError)", + "got": "KeyError)", + "got_raw": "KeyError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4827586206896552 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"table_changed_event\"", + "got": "\"stable\"", + "got_raw": "\"stable\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4827586206896552 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "add_prefix(\"pgqueuer\")", + "got": "\"test_emit_stable_changed_insert\"", + "got_raw": "\"test_emit_stable_changed_insert\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.2545454545454545 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "(10, event.root.sent_at)", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.0660670619445956, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "metrics_payload", + "got": "metrics_payload", + "got_raw": "metrics_payload", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"text/plain; charset=utf-8\"", + "got": "\"text/plain; charset=utf-8\"", + "got_raw": "\"text/plain; charset=utf-8\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "'pgqueuer_queue_count{entrypoint=\"default\",status=\"pending\"} 3'", + "got": "\"pgqueuer_queue_count{entrypoint=\\\"default\\\",status=\\\"pending\\\"} 3\\n\"", + "got_raw": "\"pgqueuer_queue_count{entrypoint=\\\"default\\\",status=\\\"pending\\\"} 3\\n\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9242424242424242 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "[ 'custom_queue_count{aggregation=\"sum\",entrypoint=\"main\",status=\"queued\"} 2' ]", + "got": "[(\"custom_queue_count\", 2)]", + "got_raw": "[(\"custom_queue_count\", 2)]", + "exact_match": false, + "code_bleu": 0.20556680845025982, + "edit_similarity": 0.4528301886792453 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "[ 'custom_log_count{aggregation=\"sum\",entrypoint=\"main\",status=\"successful\"} 5' ]", + "got": "[ 'custom_log_count{aggregation=\"sum\",entrypoint=\"main\",status=\"successful\"} 5' ]", + "got_raw": "[ 'custom_log_count{aggregation=\"sum\",entrypoint=\"main\",status=\"successful\"} 5' ]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "N", + "got": "N", + "got_raw": "N", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "set(dir(Queries))", + "got": "set(Queries.__dict__)", + "got_raw": "set(Queries.__dict__)", + "exact_match": false, + "code_bleu": 0.31333304816101065, + "edit_similarity": 0.631578947368421 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "set(dir(SentryTracing))", + "got": "set(dir(SentryTracing))", + "got_raw": "set(dir(SentryTracing))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "set(dir(LogfireTracing))", + "got": "set(type(Tracing).mro())", + "got_raw": "set(type(Tracing).mro())", + "exact_match": false, + "code_bleu": 0.1972940627795883, + "edit_similarity": 0.5833333333333334 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "set(dir(InMemoryQueries))", + "got": "set(InMemoryQueries.__dict__)", + "got_raw": "set(InMemoryQueries.__dict__)", + "exact_match": false, + "code_bleu": 0.31333304816101065, + "edit_similarity": 0.7407407407407407 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "N", + "got": "N", + "got_raw": "N", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"job_fetch\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"pick_local\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "list(range(N))", + "got": "list(range(N))", + "got_raw": "list(range(N))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "[\"to_be_picked\"] * N", + "got": "[\"to_be_picked\"] * N", + "got_raw": "[\"to_be_picked\"] * N", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "max_concurrent_tasks", + "got": "min(max_concurrent_tasks", + "got_raw": "min(max_concurrent_tasks, N)", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "N", + "got": "N", + "got_raw": "N", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "backoff.start_delay", + "got": "timedelta(milliseconds=1)", + "got_raw": "timedelta(milliseconds=1)", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.22727272727272727 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "timedelta(milliseconds=2)", + "got": "timedelta(milliseconds=8)", + "got_raw": "timedelta(milliseconds=8)", + "exact_match": false, + "code_bleu": 0.6434588841607617, + "edit_similarity": 0.96 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "[timedelta(milliseconds=2)]", + "got": "[timedelta(milliseconds=2)]", + "got_raw": "[timedelta(milliseconds=2)]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "timedelta(milliseconds=2).total_seconds())", + "got": "2.0)]", + "got_raw": "2.0)]", + "exact_match": false, + "code_bleu": 0.12321845503770609, + "edit_similarity": 0.1276595744680851 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "[pytest.approx(timedelta(milliseconds=2).total_seconds())]", + "got": "[0.02]", + "got_raw": "[0.02]", + "exact_match": false, + "code_bleu": 0.08823529411764705, + "edit_similarity": 0.125 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "N", + "got": "N", + "got_raw": "N", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "v[1]", + "got": "v[1]", + "got_raw": "v[1]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "v[0]", + "got": "v[0]", + "got_raw": "v[0]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "job_id", + "got": "job_id", + "got_raw": "job_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"picked\"", + "got": "\"picked\"", + "got_raw": "\"picked\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "len(jobs)", + "got": "len(jobs)", + "got_raw": "len(jobs)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "entrypoint", + "got": "entrypoint", + "got_raw": "entrypoint", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "expression", + "got": "expression", + "got_raw": "expression", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"1 * * * *\"", + "got": "\"1 * * * *\"", + "got_raw": "\"1 * * * *\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"2 * * * *\"", + "got": "\"2 * * * *\"", + "got_raw": "\"2 * * * *\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"3 * * * *\"", + "got": "\"3 * * * *\"", + "got_raw": "\"3 * * * *\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"sample_task\"", + "got": "\"sample_task\"", + "got_raw": "\"sample_task\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "n_tasks", + "got": "n_tasks", + "got_raw": "n_tasks", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "RuntimeError)", + "got": "FailingListenerError)", + "got_raw": "FailingListenerError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "shutdown_event", + "got": "shutdown_event", + "got_raw": "shutdown_event", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "signal.SIGINT)", + "got": "ANY)", + "got_raw": "ANY)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "signal.SIGTERM)", + "got": "signal.SIGTERM)", + "got_raw": "signal.SIGTERM)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "caplog.text.lower()", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": false, + "code_bleu": 0.36038417796587763, + "edit_similarity": 0.7333333333333333 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "FailingListenerError)", + "got": "FailingListenerError)", + "got_raw": "FailingListenerError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "Exception, match=\"Simulated failure\")", + "got": "Exception)", + "got_raw": "Exception)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.425531914893617 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "ValueError, match=\"'restart_delay' must be >= 0\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.36666666666666664 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "NotImplementedError, match=\"Unsupported instance type: .*InvalidManager.*\")", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.17909498863725698, + "edit_similarity": 0.18823529411764706 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "NotImplementedError, match=r\"Unsupported instance type: .*InvalidManager.*\")", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.153509990260506, + "edit_similarity": 0.18604651162790697 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "RuntimeError, match=r\"max_concurrent_tasks must be at least twice the batch size\")", + "got": "asyncio.Event() as shutdown_event:", + "got_raw": "asyncio.Event() as shutdown_event:", + "exact_match": false, + "code_bleu": 0.16036590969929357, + "edit_similarity": 0.20689655172413793 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "N", + "got": "N", + "got_raw": "N", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "[True] * N", + "got": "[True]", + "got_raw": "[True]", + "exact_match": false, + "code_bleu": 0.5045378491522287, + "edit_similarity": 0.75 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "N", + "got": "N", + "got_raw": "N", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "N", + "got": "N", + "got_raw": "N", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "5", + "got": "0.1", + "got_raw": "0.1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "now", + "got": "now", + "got_raw": "now", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "job_id", + "got": "job_id", + "got_raw": "job_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "additional", + "got": "additional", + "got_raw": "additional", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "\"ValueError\"", + "got": "ValueError", + "got_raw": "ValueError", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "expected_keys", + "got": "expected_keys", + "got_raw": "expected_keys", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "record.exception_message", + "got": "record.exception", + "got_raw": "record.exception", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.8 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "janbjorge/pgqueuer", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "expected_vals", + "got": "expected_vals", + "got_raw": "expected_vals", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "n_expected_vals", + "got": "n_expected_vals", + "got_raw": "n_expected_vals", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "n_expected_returns", + "got": "n_expected_returns", + "got_raw": "n_expected_returns", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "sorted(set(ret_list))", + "got": "ret_list", + "got_raw": "ret_list", + "exact_match": false, + "code_bleu": 0.08494336535733721, + "edit_similarity": 0.5517241379310345 + }, + { + "repo": "jazzband/Watson", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "\"a-project\"", + "got": "\"b-project\"", + "got_raw": "\"b-project\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "jazzband/Watson", + "expected": "\"b-project\"", + "got": "\"b-project\"", + "got_raw": "\"b-project\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "''", + "got": "''", + "got_raw": "''", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "42", + "got": "42", + "got_raw": "42", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "23", + "got": "23", + "got_raw": "23", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "'42'", + "got": "\"42\"", + "got_raw": "\"42\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "jazzband/Watson", + "expected": "3.14", + "got": "3.14", + "got_raw": "3.14", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "aniceday_lastMoon", + "got": "aniceday_lastMoon", + "got_raw": "aniceday_lastMoon", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "thenextday_lastMoon", + "got": "thenextday_lastMoon", + "got_raw": "thenextday_lastMoon", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "''", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "jazzband/Watson", + "expected": "'A'", + "got": "'-'", + "got_raw": "'-'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "jazzband/Watson", + "expected": "'B'", + "got": "'B'", + "got_raw": "'B'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "'[]'", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "jazzband/Watson", + "expected": "keys", + "got": "keys", + "got_raw": "keys", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "'foo'", + "got": "'foo'", + "got_raw": "'foo'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "header", + "got": "header", + "got_raw": "header", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "'A, B'", + "got": "'A,B'", + "got_raw": "'A,B'", + "exact_match": true, + "code_bleu": 0.5, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "jazzband/Watson", + "expected": "Abort)", + "got": "Abort)", + "got_raw": "Abort)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/Watson", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "atol)", + "got": "atol)", + "got_raw": "atol)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "kapre_again)", + "got": "kapre_again)", + "got_raw": "kapre_again)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "RuntimeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "input_shape[axis])", + "got": "input_shape[axis])", + "got_raw": "input_shape[axis])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "NotImplementedError)", + "got": "NotImplementedError)", + "got_raw": "NotImplementedError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "spec_augmented[0].shape)", + "got": "spec_augmented.shape[1:])", + "got_raw": "spec_augmented.shape[1:])", + "exact_match": false, + "code_bleu": 0.26084743001221455, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "mel_fb)", + "got": "mel_fb)", + "got_raw": "mel_fb)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "mu_src_recon)", + "got": "mu_src_recon)", + "got_raw": "mu_src_recon)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "x_decibel_ref)", + "got": "x_decibel_ref)", + "got_raw": "x_decibel_ref)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "(n_freq, n_mels)", + "got": "(n_mels, n_freq)", + "got_raw": "(n_mels, n_freq)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.5 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "(n_freq, n_bins)", + "got": "(n_freq, n_bins)", + "got_raw": "(n_freq, n_bins)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "KPB._get_floatx()", + "got": "mel_fb_ref.dtype", + "got_raw": "mel_fb_ref.dtype", + "exact_match": false, + "code_bleu": 0.2710806010829534, + "edit_similarity": 0.24242424242424243 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "mu_src_kapre.numpy())", + "got": "mu_src_kapre)", + "got_raw": "mu_src_kapre)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.7647058823529411 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "frames_ref)", + "got": "frames_ref)", + "got_raw": "frames_ref)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "energies_ref)", + "got": "energies_ref)", + "got_raw": "energies_ref)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "mfcc_kapre[1:])", + "got": "mfcc_kapre[1:])", + "got_raw": "mfcc_kapre[1:])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "mfcc_kapre[0] / np.sqrt(2.0))", + "got": "mfcc_kapre[0])", + "got_raw": "mfcc_kapre[0])", + "exact_match": false, + "code_bleu": 0.3313628036529378, + "edit_similarity": 0.6511627906976745 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "S)", + "got": "np.abs(S))", + "got_raw": "np.abs(S))", + "exact_match": false, + "code_bleu": 0.24446151121745047, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "S_lite)", + "got": "S_lite)", + "got_raw": "S_lite)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "S_phase)", + "got": "S_phase)", + "got_raw": "S_phase)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "np.sin(b))", + "got": "np.sin(b))", + "got_raw": "np.sin(b))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "np.cos(b))", + "got": "np.cos(b))", + "got_raw": "np.cos(b))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "np.abs(b))", + "got": "np.abs(b))", + "got_raw": "np.abs(b))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "delta_ref)", + "got": "delta_ref)", + "got_raw": "delta_ref)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "np.real(b))", + "got": "np.real(b))", + "got_raw": "np.real(b))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "np.imag(b))", + "got": "np.imag(b))", + "got_raw": "np.imag(b))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "np.shape(b))", + "got": "np.shape(b))", + "got_raw": "np.shape(b))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "recon_S.shape)", + "got": "recon_S.shape)", + "got_raw": "recon_S.shape)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "S_approx_phase)", + "got": "S_approx_phase)", + "got_raw": "S_approx_phase)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "recon_waveform)", + "got": "recon_waveform)", + "got_raw": "recon_waveform)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "x_concat.shape)", + "got": "tuple(x_concat.shape))", + "got_raw": "tuple(x_concat.shape))", + "exact_match": false, + "code_bleu": 0.5169731539571706, + "edit_similarity": 0.8108108108108109 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "other_channels)", + "got": "other_channels)", + "got_raw": "other_channels)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "mag_phase_ref.shape)", + "got": "mag_phase_ref.shape)", + "got_raw": "mag_phase_ref.shape)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "keunwoochoi/kapre", + "expected": "acceptable_fail_ratio", + "got": "count_failed", + "got_raw": "count_failed / len(a))", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.42424242424242425 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "deprecated.__doc__", + "got": "deprecated.__doc__", + "got_raw": "deprecated.__doc__", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "version.__class__.__name__", + "got": "deprecated.__version__", + "got_raw": "deprecated.__version__", + "exact_match": false, + "code_bleu": 0.2710806010829534, + "edit_similarity": 0.375 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "__file__", + "got": "__file__", + "got_raw": "__file__", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "demo_lineno + 1", + "got": "demo_lineno", + "got_raw": "demo_lineno", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.8461538461538461 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "str(warn.message)", + "got": "warn.message", + "got_raw": "warn.message", + "exact_match": false, + "code_bleu": 0.42044820762685725, + "edit_similarity": 0.8275862068965517 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "use_foo_lineno + 1", + "got": "use_foo_lineno", + "got_raw": "use_foo_lineno", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.875 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "u\"I am deprecated!\"", + "got": "u\"I am deprecated!\"", + "got_raw": "u\"I am deprecated!\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "3.14", + "got": "3.14", + "got_raw": "3.14", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "obj1", + "got": "obj1", + "got_raw": "obj1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "\"five\"", + "got": "\"five\"", + "got_raw": "\"five\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "\"\\n\"", + "got": "''", + "got_raw": "''", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "actual", + "got": "actual", + "got_raw": "actual", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "str(warn.message)", + "got": "warn.message", + "got_raw": "warn.message", + "exact_match": false, + "code_bleu": 0.42044820762685725, + "edit_similarity": 0.8275862068965517 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "u\"I am deprecated!\"", + "got": "\" I am deprecated!\"", + "got_raw": "\" I am deprecated!\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "3.14", + "got": "3.14", + "got_raw": "3.14", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "obj1", + "got": "obj1", + "got_raw": "obj1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "\"five\"", + "got": "\"five\"", + "got_raw": "\"five\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "laurent-laporte-pro/deprecated", + "expected": "[ {\"category\": V2DeprecationWarning, \"message\": \"epsilon is deprecated in version v2\"}, {\"category\": V2DeprecationWarning, \"message\": \"start is removed in version v2\"}, {\"category\": DeprecationWarning, \"message\": \"epsilon is deprecated in version v1.1\"}, ]", + "got": "[", + "got_raw": "[", + "exact_match": false, + "code_bleu": 0.018581361171917516, + "edit_similarity": 0.007782101167315175 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "repr(e)", + "got": "e", + "got_raw": "e", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.25 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "ma.keys()", + "got": "ma", + "got_raw": "ma", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "ma.items()", + "got": "ma", + "got_raw": "ma", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "len(tensors)", + "got": "len(tensors)", + "got_raw": "len(tensors)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "len(reference)", + "got": "num_parts", + "got_raw": "num_parts", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.17391304347826086 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "list(range(10))", + "got": "[10]", + "got_raw": "[10]", + "exact_match": false, + "code_bleu": 0.19362900077353815, + "edit_similarity": 0.21052631578947367 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "str(excinfo.value)", + "got": "str(excinfo)", + "got_raw": "str(excinfo)", + "exact_match": false, + "code_bleu": 0.4714045207910316, + "edit_similarity": 0.8 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "len(reference_knn)", + "got": "k", + "got_raw": "k", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.10526315789473684 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "dict(name=\"expert1\")", + "got": "{\"k\":", + "got_raw": "{\"k\": 1}", + "exact_match": false, + "code_bleu": 0.18995892141289814, + "edit_similarity": 0.16 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "dict(name=\"expert2\")", + "got": "dict(name=\"expert2\")", + "got_raw": "dict(name=\"expert2\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "len(targets_for_peer)", + "got": "len(targets_for_peer)", + "got_raw": "len(targets_for_peer)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "len(all_active_neighbors)", + "got": "num_added", + "got_raw": "num_added", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "len(routing_table.buckets)", + "got": "len(routing_table.buckets)", + "got_raw": "len(routing_table.buckets)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "DHTID.xor_distance(id2, id2)", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.0625, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "test_pb2.TestResponse(number=45)", + "got": "45", + "got_raw": "45", + "exact_match": false, + "code_bleu": 0.07432544468767006, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "test_pb2.TestResponse(number=11)", + "got": "test_pb2.TestResponse(number=10)", + "got_raw": "test_pb2.TestResponse(number=10)", + "exact_match": false, + "code_bleu": 0.7506238537503395, + "edit_similarity": 0.96875 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "test_pb2.TestResponse(number=100)", + "got": "test_pb2.TestResponse(number=100)", + "got_raw": "test_pb2.TestResponse(number=100)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "await protocol.call_ping(peer1_id)", + "got": "protocol.node_id", + "got_raw": "protocol.node_id", + "exact_match": false, + "code_bleu": 0.2548300960720116, + "edit_similarity": 0.56 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "await protocol.call_ping(peer_peer_id)", + "got": "peer_peer_id", + "got_raw": "peer_peer_id", + "exact_match": false, + "code_bleu": 0.08494336535733721, + "edit_similarity": 0.48 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "test_pb2.TestResponse(number=(i // 2) ** 2)", + "got": "test_pb2.TestRequest(number=i)", + "got_raw": "test_pb2.TestRequest(number=i)", + "exact_match": false, + "code_bleu": 0.1907082808182838, + "edit_similarity": 0.7123287671232876 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "test_pb2.TestResponse(number=(i // 2) ** 3)", + "got": "test_pb2.TestResponse(number=(i // 2) ** 2)", + "got_raw": "test_pb2.TestResponse(number=(i // 2) ** 2)", + "exact_match": false, + "code_bleu": 0.8843946454355334, + "edit_similarity": 0.9767441860465116 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "max((len(serialized_tensor.buffer) - 1) // chunk_size + 1, 1)", + "got": "len(compression)", + "got_raw": "len(compression)", + "exact_match": false, + "code_bleu": 0.10162654963092295, + "edit_similarity": 0.2077922077922078 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "x**2", + "got": "x**2", + "got_raw": "x**2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "emas[2]", + "got": "emas[2]", + "got_raw": "emas[2]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "tensor.dtype", + "got": "tensor.dtype", + "got_raw": "tensor.dtype", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "stored.value", + "got": "stored", + "got_raw": "stored", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "record.value", + "got": "record", + "got_raw": "record", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "wall_time / 3", + "got": "0.5 * wall_time", + "got_raw": "0.5 * wall_time", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.6428571428571429 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "control2.stage", + "got": "AveragingStage.STAGE_FINISHED", + "got_raw": "AveragingStage.STAGE_FINISHED", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.23255813953488372 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "p2ps[-1].peer_id", + "got": "p2ps[-1].peer_id", + "got_raw": "p2ps[-1].peer_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "plain_record.value", + "got": "plain_record", + "got_raw": "plain_record", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.8 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "other_peer.peer_id", + "got": "first_peer.peer_id", + "got_raw": "first_peer.peer_id", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "signed_record.value", + "got": "signed_record.value", + "got_raw": "signed_record.value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "step_time_deltas[i]", + "got": "emas[i]", + "got_raw": "emas[i]", + "exact_match": false, + "code_bleu": 0.668740304976422, + "edit_similarity": 0.5384615384615384 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "tensor.requires_grad", + "got": "tensor.requires_grad", + "got_raw": "tensor.requires_grad", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "n_processes * n_coros", + "got": "n_processes", + "got_raw": "n_processes", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.6875 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "server_info_2.peer_id", + "got": "server_info_2.peer_id", + "got_raw": "server_info_2.peer_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "server_info_3.peer_id", + "got": "server_info_3.peer_id", + "got_raw": "server_info_3.peer_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "remaining_peer1.peer_id", + "got": "remaining_peer1.peer_id", + "got_raw": "remaining_peer1.peer_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "EXPERT_WEIGHT_UPDATES + 1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "asyncio.IncompleteReadError)", + "got": "P2PHandlerError)", + "got_raw": "P2PHandlerError)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.45454545454545453 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "optimizer.tracker.global_epoch", + "got": "total_epochs", + "got_raw": "total_epochs", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "BACKWARD_PASSES_BEFORE_SAVE + 1", + "got": "BACKWARD_PASSES_AFTER_SAVE", + "got_raw": "BACKWARD_PASSES_AFTER_SAVE", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.8070175438596491 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "total_samples_accumulated.value", + "got": "total_samples_accumulated.value", + "got_raw": "total_samples_accumulated.value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "concurrent.futures.TimeoutError)", + "got": "asyncio.TimeoutError)", + "got_raw": "asyncio.TimeoutError)", + "exact_match": false, + "code_bleu": 0.44582686998428134, + "edit_similarity": 0.6037735849056604 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "third_validator.local_public_key", + "got": "third_validator.local_public_key", + "got_raw": "third_validator.local_public_key", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "second_validator.local_public_key", + "got": "second_validator.local_public_key", + "got_raw": "second_validator.local_public_key", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "2 if sync_epoch_when_averaging else 1", + "got": "2 + 1)", + "got_raw": "2 + 1)", + "exact_match": false, + "code_bleu": 0.3181082915068203, + "edit_similarity": 0.18604651162790697 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "4", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "4", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "7", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "0", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "0", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "4", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "0", + "got": "20", + "got_raw": "20", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "2", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "13", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "16", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "32", + "got": "4", + "got_raw": "4", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "0.5", + "got": "0.9", + "got_raw": "0.9", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "123", + "got": "\"123\"", + "got_raw": "\"123\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "456", + "got": "456", + "got_raw": "456", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "124", + "got": "124", + "got_raw": "124", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "125", + "got": "125", + "got_raw": "125", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "126", + "got": "126", + "got_raw": "126", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "-99", + "got": "-100", + "got_raw": "-100", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "777", + "got": "777", + "got_raw": "777", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "999", + "got": "999", + "got_raw": "999", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "0.0", + "got": "PEAK_LR", + "got_raw": "PEAK_LR", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "0.2", + "got": "0.2", + "got_raw": "0.2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "150", + "got": "152", + "got_raw": "152", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "321", + "got": "321", + "got_raw": "321", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "700", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "100", + "got": "100", + "got_raw": "100", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "\"\"", + "got": "\"/ip4/127.0.0.1/udp/1234\"", + "got_raw": "\"/ip4/127.0.0.1/udp/1234\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.14814814814814814 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "\"bob\"", + "got": "\"bob\"", + "got_raw": "\"bob\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "\"pew\"", + "got": "\"pew\"", + "got_raw": "\"pew\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "\"ip4\"", + "got": "\"ip4\"", + "got_raw": "\"ip4\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "\"123\"", + "got": "proto", + "got_raw": "proto", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "b\"val\"", + "got": "b\"val\"", + "got_raw": "b\"val\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "b\"pong\"", + "got": "b\"pong\"", + "got_raw": "b\"pong\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "b\"ping\"", + "got": "\"123\"", + "got_raw": "\"123\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "\"alice\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "b\"val1\"", + "got": "b\"val1\"", + "got_raw": "b\"val1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "b\"val2\"", + "got": "b\"val2\"", + "got_raw": "b\"val2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "\"5555\")", + "got": "5555)", + "got_raw": "5555)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "b\"\\x05\"", + "got": "b\"\\x05\"", + "got_raw": "b\"\\x05\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "b\"okay\"", + "got": "b\"okay\"", + "got_raw": "b\"okay\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "\"value1\"", + "got": "\"value1\"", + "got_raw": "\"value1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "b\"value\"", + "got": "record.value", + "got_raw": "record.value", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.5 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "b\"val1a2\"", + "got": "b\"val1a2\"", + "got_raw": "b\"val1a2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "b\"foo_bar\"", + "got": "b\"foo_bar\"", + "got_raw": "b\"foo_bar\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "b\"true_value\"", + "got": "b\"true_value\"", + "got_raw": "b\"true_value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "\"string_value\"", + "got": "42", + "got_raw": "42", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "None", + "got": "proto2", + "got_raw": "proto2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "d", + "got": "d", + "got_raw": "d", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "m2", + "got": "m2", + "got_raw": "m2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "m1", + "got": "m3", + "got_raw": "m3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "m3", + "got": "m3", + "got_raw": "m3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "ma", + "got": "proto2", + "got_raw": "proto2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "buf", + "got": "buf", + "got_raw": "buf", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "ref", + "got": "ref", + "got_raw": "ref", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "beta", + "got": "beta", + "got_raw": "beta", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "ours", + "got": "ours", + "got_raw": "ours", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "line", + "got": "line", + "got_raw": "line", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "alpha", + "got": "alpha", + "got_raw": "alpha", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "value", + "got": "value", + "got_raw": "value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "proto", + "got": "proto", + "got_raw": "proto", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "MADDR", + "got": "MADDR", + "got_raw": "MADDR", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "proto", + "got": "proto", + "got_raw": "proto", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "proto2", + "got": "proto2", + "got_raw": "proto2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "proto3", + "got": "proto3", + "got_raw": "proto3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "proto4", + "got": "proto4", + "got_raw": "proto4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "string", + "got": "string", + "got_raw": "string", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "PEAK_LR", + "got": "PEAK_LR", + "got_raw": "PEAK_LR", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "integer", + "got": "integer", + "got_raw": "integer", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "PEER_ID", + "got": "PEER_ID", + "got_raw": "PEER_ID", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "n_peers", + "got": "n_peers", + "got_raw": "n_peers", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "phony_id", + "got": "phony_id", + "got_raw": "phony_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "query_id", + "got": "query_id", + "got_raw": "query_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "beam_size", + "got": "beam_size", + "got_raw": "beam_size", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "ip6_bytes", + "got": "ip6_bytes", + "got_raw": "ip6_bytes", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "num_added", + "got": "num_added", + "got_raw": "num_added", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "threshold", + "got": "threshold", + "got_raw": "threshold", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "total_size", + "got": "2", + "got_raw": "2 * num_replicas", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "ip6_string", + "got": "ip6_string", + "got_raw": "ip6_string", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "TypeError)", + "got": "exceptions.InvalidProtocolCode)", + "got_raw": "exceptions.InvalidProtocolCode)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.1951219512195122 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "Exception)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "table_size", + "got": "table_size", + "got_raw": "table_size", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "second_line", + "got": "second_line", + "got_raw": "second_line", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "ValueError)", + "got": "MPFuture)", + "got_raw": "MPFuture)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "expected_key", + "got": "expected_key", + "got_raw": "expected_key", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "RuntimeError)", + "got": "RuntimeError)", + "got_raw": "RuntimeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "routing_table", + "got": "routing_table", + "got_raw": "routing_table", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "P2PDaemonError)", + "got": "P2PHandlerError)", + "got_raw": "P2PHandlerError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "first_report_msg", + "got": "first_report_msg", + "got_raw": "first_report_msg", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "P2PHandlerError)", + "got": "P2PHandlerError)", + "got_raw": "P2PHandlerError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "allow_extra_keys", + "got": "False", + "got_raw": "False", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "{}", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "(4, 3)", + "got": "(4, 5)", + "got_raw": "(4, 5)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "[MADDR]", + "got": "[MADDR]", + "got_raw": "[MADDR]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "(50, 50)", + "got": "(0, 0, 0)", + "got_raw": "(0, 0, 0)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "(1, 0, 1)", + "got": "(0, 1, 1)", + "got_raw": "(0, 1, 1)", + "exact_match": false, + "code_bleu": 0.5169731539571706, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "(1, 0, 0)", + "got": "(0, 0, 1)", + "got_raw": "(0, 0, 1)", + "exact_match": false, + "code_bleu": 0.45499414040480374, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "(41, 19, 0)", + "got": "[10, 10, 10]", + "got_raw": "[10, 10, 10]", + "exact_match": false, + "code_bleu": 0.20556680845025982, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "(1, hidden_dim)", + "got": "(1, hidden_dim)", + "got_raw": "(1, hidden_dim)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "(3, hidden_dim)", + "got": "(3, hidden_dim)", + "got_raw": "(3, hidden_dim)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "(4, 3, hidden_dim)", + "got": "(4, 3, hidden_dim)", + "got_raw": "(4, 3, hidden_dim)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "(1 / num_experts) / 2", + "got": "0.0", + "got_raw": "0.0", + "exact_match": false, + "code_bleu": 0.07142857142857142, + "edit_similarity": 0.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "(P2PDaemonError, ConnectionError))", + "got": "P2PDaemonError)", + "got_raw": "P2PDaemonError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.6122448979591837 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "[SchemaValidator, RSASignatureValidator]", + "got": "[RSASignatureValidator()]", + "got_raw": "[RSASignatureValidator()]", + "exact_match": false, + "code_bleu": 0.32466791547509893, + "edit_similarity": 0.7076923076923077 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "(protocol.serializer.dumps(value1), expiration)", + "got": "value1", + "got_raw": "value1", + "exact_match": false, + "code_bleu": 0.049550296458446705, + "edit_similarity": 0.22641509433962265 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "learning-at-home/hivemind", + "expected": "False", + "got": "b\"value4\"", + "got_raw": "b\"value4\", time + 4)", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.42857142857142855 + }, + { + "repo": "mar10/wsgidav", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "\"utf-8\"", + "got": "\"utf-8\"", + "got_raw": "\"utf-8\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "res.text", + "got": "res.text", + "got_raw": "res.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "\"litmus suite failed: check the log\")", + "got": "\"litmus failed\")", + "got_raw": "\"litmus failed\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6037735849056604 + }, + { + "repo": "mar10/wsgidav", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "url", + "got": "url", + "got_raw": "url", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "self.root", + "got": "self.root", + "got_raw": "self.root", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "__debug__)", + "got": "None)", + "got_raw": "None)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "mar10/wsgidav", + "expected": "AssertionError)", + "got": "None)", + "got_raw": "None)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3 + }, + { + "repo": "mar10/wsgidav", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "__debug__)", + "got": "None)", + "got_raw": "None)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "mar10/wsgidav", + "expected": "AssertionError)", + "got": ".pm._close()", + "got_raw": ".pm._close()", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "mar10/wsgidav", + "expected": "\"my name is joe\"", + "got": "\"my name is joe\"", + "got_raw": "\"my name is joe\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "data1", + "got": "data1", + "got_raw": "data1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "1735660800", + "got": "1735660800", + "got_raw": "1735660800", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "__debug__)", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "mar10/wsgidav", + "expected": "3)", + "got": "3)", + "got_raw": "3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "5)", + "got": "5)", + "got_raw": "5)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "42)", + "got": "7)", + "got_raw": "7)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4 + }, + { + "repo": "mar10/wsgidav", + "expected": "32)", + "got": "32)", + "got_raw": "32)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "204)", + "got": "200)", + "got_raw": "200)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.75 + }, + { + "repo": "mar10/wsgidav", + "expected": "self.SIZE)", + "got": "self.SIZE)", + "got_raw": "self.SIZE)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "ValueError)", + "got": "IOError)", + "got_raw": "IOError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.631578947368421 + }, + { + "repo": "mar10/wsgidav", + "expected": "\"further read() returns ''\")", + "got": "\"no more data\")", + "got_raw": "\"no more data\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.37209302325581395 + }, + { + "repo": "mar10/wsgidav", + "expected": "\"Read after close() returns ''\")", + "got": "\"\")", + "got_raw": "\"\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.17142857142857143 + }, + { + "repo": "mar10/wsgidav", + "expected": "\"last chunk delivers the reminder\")", + "got": "\"Read 2 bytes\")", + "got_raw": "\"Read 2 bytes\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.24 + }, + { + "repo": "mar10/wsgidav", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "[]", + "got": "()", + "got_raw": "()", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "baseOutput", + "got": "baseOutput", + "got_raw": "baseOutput", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "__debug__)", + "got": "None)", + "got_raw": "None)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "mar10/wsgidav", + "expected": "res", + "got": "res.body", + "got_raw": "res.body", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "mar10/wsgidav", + "expected": "data", + "got": "data", + "got_raw": "data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "data1", + "got": "data1", + "got_raw": "data1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "data2", + "got": "data2", + "got_raw": "data2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "data3", + "got": "data3", + "got_raw": "data3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mar10/wsgidav", + "expected": "__debug__)", + "got": "None)", + "got_raw": "None)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "mar10/wsgidav", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "llm", + "got": "llm", + "got_raw": "llm", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "llm", + "got": "llm", + "got_raw": "llm", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "val", + "got": "val", + "got_raw": "val", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "body", + "got": "body", + "got_raw": "body", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "output", + "got": "output", + "got_raw": "output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "output", + "got": "output", + "got_raw": "output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "output", + "got": "output", + "got_raw": "output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "fake_llm", + "got": "fake_llm", + "got_raw": "fake_llm", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "EXAMPLES", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "template", + "got": "template", + "got_raw": "template", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "KeyError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "loaded_llm", + "got": "loaded_llm", + "got_raw": "loaded_llm", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "loaded_llm)", + "got": "loaded_llm)", + "got_raw": "loaded_llm)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "loaded_llm)", + "got": "loaded_llm)", + "got_raw": "loaded_llm)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "RuntimeError)", + "got_raw": "RuntimeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "loaded_llm)", + "got": "loaded_llm)", + "got_raw": "loaded_llm)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "loaded_llm)", + "got": "loaded_llm)", + "got_raw": "loaded_llm)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "loaded_llm)", + "got": "loaded_llm)", + "got_raw": "loaded_llm)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "KeyError)", + "got_raw": "KeyError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "KeyError)", + "got_raw": "KeyError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "second_output", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "second_output", + "got": "second_output", + "got_raw": "second_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "second_output", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "RuntimeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "expected_docs", + "got": "expected_docs", + "got_raw": "expected_docs", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "simple_prompt", + "got": "simple_prompt", + "got_raw": "simple_prompt", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "fake_llm_chain", + "got": "fake_llm_chain", + "got_raw": "fake_llm_chain", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "expected_output", + "got": "expected_output", + "got_raw": "expected_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "expected_output", + "got": "expected_output", + "got_raw": "expected_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "expected_output", + "got": "expected_output", + "got_raw": "expected_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "expected_output", + "got": "expected_output", + "got_raw": "expected_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "expected_output", + "got": "expected_output", + "got_raw": "expected_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "expected_output", + "got": "expected_output", + "got_raw": "expected_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "expected_output", + "got": "expected_output", + "got_raw": "expected_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "expected_output", + "got": "expected_output", + "got_raw": "expected_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "AssertionError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5384615384615384 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "expected_result", + "got": "expected_result", + "got_raw": "expected_result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "expected_output", + "got": "expected_output", + "got_raw": "expected_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "expected_output", + "got": "expected_output", + "got_raw": "expected_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "expected_output", + "got": "expected_output", + "got_raw": "expected_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "expected_output", + "got": "expected_output", + "got_raw": "expected_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "expected_output", + "got": "expected_output", + "got_raw": "expected_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[docs]", + "got": "[docs]", + "got_raw": "[docs]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "{\"foo\": 3}", + "got": "{\"foo\": 3}", + "got_raw": "{\"foo\": 3}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "{\"foo\": 3}", + "got": "{\"foo\": 3}", + "got_raw": "{\"foo\": 3}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "{\"baz\": \"\"}", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "{\"baz\": \"\"}", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.23784142300054423, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[compare_run]", + "got": "[compare_run]", + "got_raw": "[compare_run]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[\"foo\", \"bar\"]", + "got": "[\"foo\", \"bar\"]", + "got_raw": "[\"foo\", \"bar\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[\"foo\", \"bar\"]", + "got": "\"foo bar\"", + "got_raw": "\"foo bar\"", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.782608695652174 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[compare_run] * 3", + "got": "[compare_run]", + "got_raw": "[compare_run]", + "exact_match": false, + "code_bleu": 0.5045378491522287, + "edit_similarity": 0.8666666666666667 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[True, True, True]", + "got": "[True, True, True]", + "got_raw": "[True, True, True]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "{\"foo\": 3, \"bar\": 2}", + "got": "{\"foo\": 3, \"bar\": 2}", + "got_raw": "{\"foo\": 3, \"bar\": 2}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "{\"foo\": 3, \"bar\": 2}", + "got": "{\"foo\": 3, \"bar\": 2}", + "got_raw": "{\"foo\": 3, \"bar\": 2}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[_get_compare_run()]", + "got": "[", + "got_raw": "[", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[\"Human: bar\\nAI: foo\"]", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.35714285714285715 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[compare_run] * num_runs", + "got": "[num_runs]", + "got_raw": "[num_runs]", + "exact_match": false, + "code_bleu": 0.32237097954706256, + "edit_similarity": 0.5294117647058824 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[\"Human: bar1\\nAI: foo1\"]", + "got": "\"foo\\nfoo1\"", + "got_raw": "\"foo\\nfoo1\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[EXAMPLES[2], EXAMPLES[0]]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.1081097377275201, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "{\"foo\": \"bar\", \"bar\": \"baz\"}", + "got": "{\"foo\": \"bar\"}", + "got_raw": "{\"foo\": \"bar\"}", + "exact_match": false, + "code_bleu": 0.44181707153725036, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "{\"baz\": \"Human: bar\\nAI: foo\"}", + "got": "{\"baz\": \"Human: bar\\nAI: foo\"}", + "got_raw": "{\"baz\": \"Human: bar\\nAI: foo\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[Document(page_content=\"foo\")]", + "got": "[Document(page_content=\"foo\"", + "got_raw": "[Document(page_content=\"foo\", embedding=[0.1, 0.2])]", + "exact_match": false, + "code_bleu": 0.75, + "edit_similarity": 0.9655172413793104 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[Document(page_content=\"foo\")]", + "got": "[Document(page_content=\"foo\"", + "got_raw": "[Document(page_content=\"foo\", embedding=[0.2, 0.1])", + "exact_match": false, + "code_bleu": 0.75, + "edit_similarity": 0.9655172413793104 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[Document(page_content=\"foo\")]", + "got": "[Document(page_content=\"foo\")]", + "got_raw": "[Document(page_content=\"foo\")]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[Document(page_content=\"foo\")]", + "got": "[Document(page_content=\"foo\"", + "got_raw": "[Document(page_content=\"foo\", metadata={\"foo\": \"bar\"})]", + "exact_match": false, + "code_bleu": 0.75, + "edit_similarity": 0.9655172413793104 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[Document(page_content=\"foo\")]", + "got": "[Document(page_content=\"foo\"", + "got_raw": "[Document(page_content=\"foo\", embedding=None)]", + "exact_match": false, + "code_bleu": 0.75, + "edit_similarity": 0.9655172413793104 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[Document(page_content=\"bar\")]", + "got": "[Document(page_content=\"foo\"", + "got_raw": "[Document(page_content=\"foo\", embedding=None)]", + "exact_match": false, + "code_bleu": 0.6068300336776659, + "edit_similarity": 0.8620689655172413 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[Document(page_content=\"baz\")]", + "got": "[Document(page_content=\"foo\"", + "got_raw": "[Document(page_content=\"foo\", embedding=None)]", + "exact_match": false, + "code_bleu": 0.6068300336776659, + "edit_similarity": 0.8620689655172413 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[Document(page_content=\"foo\")]", + "got": "[Document(page_content=\"foo\"", + "got_raw": "[Document(page_content=\"foo\", metadata={\"test\": \"test\"})]", + "exact_match": false, + "code_bleu": 0.75, + "edit_similarity": 0.9655172413793104 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[Document(page_content=\"foo\")]", + "got": "[Document(page_content=\"foo\"", + "got_raw": "[Document(page_content=\"foo\", embedding=FakeEmbeddings().embed(\"foo\"))]", + "exact_match": false, + "code_bleu": 0.75, + "edit_similarity": 0.9655172413793104 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "{\"bar\": \"baz\", \"text1\": \"foo\"}", + "got": "{\"text1\": \"This is a baz:\"}", + "got_raw": "{\"text1\": \"This is a baz:\"}", + "exact_match": false, + "code_bleu": 0.22680460581325723, + "edit_similarity": 0.45614035087719296 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[Document(page_content=\"test\")]", + "got": "[Document(page_content=\"test\"", + "got_raw": "[Document(page_content=\"test\", metadata={\"page\": 0})]", + "exact_match": false, + "code_bleu": 0.75, + "edit_similarity": 0.9666666666666667 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[_get_compare_run()] * num_threads", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.16988673071467442, + "edit_similarity": 0.1111111111111111 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "{\"baz\": \"foo\\nHuman: bar1\\nAI: foo1\"}", + "got": "{ \"baz\": \"Human: bar1\\nAI: foo1\" }", + "got_raw": "{ \"baz\": \"Human: bar1\\nAI: foo1\" }", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.9014084507042254 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[EXAMPLES[2], EXAMPLES[0], EXAMPLES[1]]", + "got": "[\"foo1\", \"foo3\"]", + "got_raw": "[\"foo1\", \"foo3\"]", + "exact_match": false, + "code_bleu": 0.10145872358596841, + "edit_similarity": 0.14545454545454545 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "{\"bar\": \"baz\", \"stop\": [\"foo\"], \"text1\": \"bar\"}", + "got": "{\"bar\": \"baz\", \"text1\": \"baz\"}", + "got_raw": "{\"bar\": \"baz\", \"text1\": \"baz\"}", + "exact_match": false, + "code_bleu": 0.3948222038857477, + "edit_similarity": 0.7532467532467533 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[Document(page_content=\"foo\", metadata={\"page\": 0})]", + "got": "[Document(page_content=\"foo\", metadata={\"page\": 0})]", + "got_raw": "[Document(page_content=\"foo\", metadata={\"page\": 0})]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[Document(page_content=\"foo\", metadata={\"page\": 0})]", + "got": "[Document(page_content=\"foo\", metadata={\"page\": 0})]", + "got_raw": "[Document(page_content=\"foo\", metadata={\"page\": 0})]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[Document(page_content=\"foo\", metadata={\"page\": 0})]", + "got": "[Document(page_content=\"foo\", metadata={\"page\": 0})]", + "got_raw": "[Document(page_content=\"foo\", metadata={\"page\": 0})]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[Document(page_content=\"foo\", metadata={\"page\": 0})]", + "got": "[Document(page_content=\"foo\", metadata={\"page\": 0})]", + "got_raw": "[Document(page_content=\"foo\", metadata={\"page\": 0})]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[Document(page_content=\"foo\", metadata={\"page\": 0})]", + "got": "[Document(page_content=\"foo\", metadata={\"page\": 0})]", + "got_raw": "[Document(page_content=\"foo\", metadata={\"page\": 0})]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "[Document(page_content=\"foo\", metadata={\"page\": \"0\"})]", + "got": "[Document(page_content=\"foo\", metadata={\"page\": \"0\"})]", + "got_raw": "[Document(page_content=\"foo\", metadata={\"page\": \"0\"})]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "docs[0]", + "got": "(\"foo\"", + "got_raw": "(\"foo\", {})", + "exact_match": false, + "code_bleu": 0.2259005009024612, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "scores[1]", + "got": "scores[1]", + "got_raw": "scores[1]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "scores[1]", + "got": "scores[1]", + "got_raw": "scores[1]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "EXAMPLES[:1]", + "got": "[{\"question\":", + "got_raw": "[{\"question\": \"Question: who are you?\\nAnswer: foo\"}]", + "exact_match": false, + "code_bleu": 0.3181082915068203, + "edit_similarity": 0.16 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "docs[0].metadata", + "got": "docs[0].metadata", + "got_raw": "docs[0].metadata", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "docs[0].page_content", + "got": "docs[0].page", + "got_raw": "docs[0].page", + "exact_match": false, + "code_bleu": 0.8091067115702212, + "edit_similarity": 0.75 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "EXAMPLES + [new_example]", + "got": "[short_question]", + "got_raw": "[short_question]", + "exact_match": false, + "code_bleu": 0.29999999999999993, + "edit_similarity": 0.2 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "expected_docstore.__dict__", + "got": "expected_docstore.__dict__", + "got_raw": "expected_docstore.__dict__", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "test_api_data[\"api_summary\"]", + "got": "test_api_data[\"api_response\"]", + "got_raw": "test_api_data[\"api_response\"]", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7719298245614035 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "prompt_from_template.template", + "got": "prompt_from_template.template", + "got_raw": "prompt_from_template.template", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "prompt_from_template.input_variables", + "got": "prompt_from_template.input_variables", + "got_raw": "prompt_from_template.input_variables", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"2\"", + "got": "\"1", + "got_raw": "\"1 purple item remains on the desk.\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.4 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"28\"", + "got": "\"Jan", + "got_raw": "\"Jan has 12 pets. \"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.25 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"2\\n\"", + "got": "\"2\"", + "got_raw": "\"2\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"NBA\"", + "got": "\"NBA\"", + "got_raw": "\"NBA\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"baz\"", + "got": "\"bar\"", + "got_raw": "\"bar\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"foo\"", + "got": "\"this", + "got_raw": "\"this is a baz:\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.2 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"2\\n\"", + "got": "\"```bash\\nexpr", + "got_raw": "\"```bash\\nexpr 1 + 1\\n```\"", + "exact_match": false, + "code_bleu": 0.1561969968460128, + "edit_similarity": 0.3157894736842105 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"bar\"", + "got": "url", + "got_raw": "url", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.25 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"bar\"", + "got": "\"bar\"", + "got_raw": "\"bar\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"1994\"", + "got": "\"1994\"", + "got_raw": "\"1994\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"2022\"", + "got": "\"2022\"", + "got_raw": "\"2022\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"foo \"", + "got": "\"foo \"", + "got_raw": "\"foo \"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"Albany\"", + "got": "\"New", + "got_raw": "\"New York is the capital of the United States.\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"Albany\"", + "got": "\"New", + "got_raw": "\"New York is the capital of the United States.\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"Albany\"", + "got": "\"New", + "got_raw": "\"New York\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"Search\"", + "got": "\"Search\"", + "got_raw": "\"Search\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"search\"", + "got": "\"search\"", + "got_raw": "\"search\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"foo bar \"", + "got": "\"foo bar \"", + "got_raw": "\"foo bar \"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"test_name\"", + "got": "\"test_name\"", + "got_raw": "\"test_name\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"Answer: 2\"", + "got": "\"Answer: 2\"", + "got_raw": "\"Answer: 2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"1994\\n1993\"", + "got": "\"1994\\n1993\"", + "got_raw": "\"1994\\n1993\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"search_api\"", + "got": "\"Search", + "got_raw": "\"Search API\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.631578947368421 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"API result\"", + "got": "\"API result\"", + "got_raw": "\"API result\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"No Results\"", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"hello world\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"foo bar baz\"", + "got": "text", + "got_raw": "text", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"Bill Clinton\"", + "got": "\"David Chanoff has collaborated with Admiral Grace Hopper", + "got_raw": "\"David Chanoff has collaborated with Admiral Grace Hopper, who served as the ambassador to the United Kingdom under President Harry S. Truman.\"", + "exact_match": false, + "code_bleu": 0.11868405219520975, + "edit_similarity": 0.16901408450704225 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"misalignment\"", + "got": "\"FooBarBaz\"", + "got_raw": "\"FooBarBaz\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.24 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"Final Answer\"", + "got": "\"Search\"", + "got_raw": "\"Search\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"No More Results\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.21052631578947367 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"El Palmar, Spain\"", + "got": "\"Answer: The hometown of the reigning men's U.S. Open champion is not mentioned in the provided prompt.\"", + "got_raw": "\"Answer: The hometown of the reigning men's U.S. Open champion is not mentioned in the provided prompt.\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.09836065573770492 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"test_description\"", + "got": "\"test_description\"", + "got_raw": "\"test_description\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"This text is bad.\"", + "got": "critique[\"text\"]", + "got_raw": "critique[\"text\"]", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.34285714285714286 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"ID bar not found.\"", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "f\"Answer: {2**.5}\\n\"", + "got": "\"Answer: 1.414213562373095\"", + "got_raw": "\"Answer: 1.414213562373095\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.5106382978723404 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"curses foiled again\"", + "got": "\"BadAction\\nAction: Final Answer\\nAction Input: curses foiled again\"", + "got_raw": "\"BadAction\\nAction: Final Answer\\nAction Input: curses foiled again\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.47191011235955055 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"curses foiled again\"", + "got": "\"Action 1: BadAction[langchain]", + "got_raw": "\"Action 1: BadAction[langchain]", + "exact_match": false, + "code_bleu": 0.13485111859503685, + "edit_similarity": 0.34615384615384615 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"I still don't know.\"", + "got": "question", + "got_raw": "question", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3448275862068966 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"file1.txt\\nfile2.txt\"", + "got": "\"file1.txt\\nfile2.txt\"", + "got_raw": "\"file1.txt\\nfile2.txt\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"\\nFriend: bar\\nAI: foo\"", + "got": "\"Friend: bar\\nFriend: foo\"", + "got_raw": "\"Friend: bar\\nFriend: foo\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"Barack Hussein Obama II\"", + "got": "\"Barack\"", + "got_raw": "\"Barack\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.48484848484848486 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"name 'foo' is not defined\"", + "got": "\"3\\n\"", + "got_raw": "\"3\\n\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.1875 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"\\nHuman: bar\\nAssistant: foo\"", + "got": "[\"Assistant: foo\"", + "got_raw": "[\"Assistant: foo\", \"bar: foo\"]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.6808510638297872 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "\"Question: {question}\\nAnswer:\"", + "got": "\"Question: {question}\"", + "got_raw": "\"Question: {question}\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8301886792452831 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "8", + "got": "8", + "got_raw": "8", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "2", + "got": "handler.tool_ends", + "got_raw": "handler.tool_ends", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "7", + "got": "handler.ends", + "got_raw": "handler.ends", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "1", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "3", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "1", + "got": "[(\"run\"", + "got_raw": "[(\"run\", \"bar\")]", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "1", + "got": "[(\"foo\"", + "got_raw": "[(\"foo\", \"foo\")]", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "768", + "got": "768", + "got_raw": "768", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "768", + "got": "768", + "got_raw": "768", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "768", + "got": "768", + "got_raw": "768", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "len(texts)", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "len(text_input)", + "got": "len(text_input)", + "got_raw": "len(text_input)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "type(loaded_llm)", + "got": "type(loaded_llm)", + "got_raw": "type(loaded_llm)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "sorted(expected_output.split())", + "got": "sorted(expected_output.split())", + "got_raw": "sorted(expected_output.split())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "sorted(\" \".join(expected_output.split()))", + "got": "sorted(expected_output.split())", + "got_raw": "sorted(expected_output.split())", + "exact_match": false, + "code_bleu": 0.5657370555230268, + "edit_similarity": 0.8611111111111112 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "sorted(\" \".join(expected_output.split()))", + "got": "sorted(\" \".join(expected_output.split()))", + "got_raw": "sorted(\" \".join(expected_output.split()))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "subprocess.check_output(\"pwd\", shell=True).decode()", + "got": "\"~\"", + "got_raw": "\"~\"", + "exact_match": false, + "code_bleu": 0.03571428571428571, + "edit_similarity": 0.07407407407407407 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/MM-REACT", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "storage.read()", + "got": "storage.read()", + "got_raw": "storage.read()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "[{'key': 'value'}]", + "got": "[{'key': 'value'}]", + "got_raw": "[{'key': 'value'}]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "6", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "0", + "got": "-1", + "got_raw": "-1", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "-4", + "got": "-5", + "got_raw": "-5", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "msiemens/tinydb", + "expected": "db.get(where('char') == 'a')", + "got": "db", + "got_raw": "db", + "exact_match": false, + "code_bleu": 0.049550296458446705, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "msiemens/tinydb", + "expected": "d", + "got": "d", + "got_raw": "d", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "0", + "got": "(q.foo", + "got_raw": "(q.foo == 3)", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "''", + "got": "''", + "got_raw": "''", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "\"Query()\"", + "got": "\"\"", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "msiemens/tinydb", + "expected": "ValueError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "msiemens/tinydb", + "expected": "hash(where('asd'))", + "got": "hash(where('value'))", + "got_raw": "hash(where('value'))", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.8421052631578947 + }, + { + "repo": "msiemens/tinydb", + "expected": "hash(where('value'))", + "got": "hash(query({'value':", + "got_raw": "hash(query({'value': 10}))", + "exact_match": false, + "code_bleu": 0.2777619034011791, + "edit_similarity": 0.75 + }, + { + "repo": "msiemens/tinydb", + "expected": "where('value').exists()", + "got": "where('value')", + "got_raw": "where('value')", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7567567567567568 + }, + { + "repo": "msiemens/tinydb", + "expected": "where('value')['val'].exists()", + "got": "where('value').has('val')", + "got_raw": "where('value').has('val')", + "exact_match": false, + "code_bleu": 0.37226793305847583, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "msiemens/tinydb", + "expected": "\"QueryImpl('==', ('type',), 'peach')\"", + "got": "\"Query(type='peach')\"", + "got_raw": "\"Query(type='peach')\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6896551724137931 + }, + { + "repo": "msiemens/tinydb", + "expected": "RuntimeError, match='Empty query was evaluated')", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.17909498863725698, + "edit_similarity": 0.2711864406779661 + }, + { + "repo": "msiemens/tinydb", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "item", + "got": "item", + "got_raw": "item", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "item2", + "got": "item2", + "got_raw": "item2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "[doc]", + "got": "[doc]", + "got_raw": "[doc]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "{'a': 1}", + "got": "{'a': 1}", + "got_raw": "{'a': 1}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "IOError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "msiemens/tinydb", + "expected": "exception)", + "got": "exception)", + "got_raw": "exception)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "storage.read()", + "got": "storage.read()", + "got_raw": "storage.read()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "[]", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "100", + "got": "100", + "got_raw": "100", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "[\"a\"]", + "got": "[\"a\"", + "got_raw": "[\"a\", \"a\", \"a\"]", + "exact_match": false, + "code_bleu": 0.4714045207910317, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "msiemens/tinydb", + "expected": "[\"b\"]", + "got": "[\"b\"", + "got_raw": "[\"b\", \"c\"]", + "exact_match": false, + "code_bleu": 0.4714045207910317, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "msiemens/tinydb", + "expected": "cache", + "got": "cache", + "got_raw": "cache", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "msiemens/tinydb", + "expected": "[\"c\", \"a\", \"d\"]", + "got": "[\"c\", \"b\", \"a\"]", + "got_raw": "[\"c\", \"b\", \"a\"]", + "exact_match": false, + "code_bleu": 0.42728700639623407, + "edit_similarity": 0.8666666666666667 + }, + { + "repo": "msiemens/tinydb", + "expected": "[\"a\", \"b\", \"c\"]", + "got": "[\"b\", \"c\", \"a\"]", + "got_raw": "[\"b\", \"c\", \"a\"]", + "exact_match": false, + "code_bleu": 0.4111336169005197, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "msiemens/tinydb", + "expected": "[\"b\", \"c\", \"a\"]", + "got": "[\"b\", \"c\", \"a\"]", + "got_raw": "[\"b\", \"c\", \"a\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "css", + "got": "css", + "got_raw": "css", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "expected_css", + "got": "expected_css", + "got_raw": "expected_css", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "expected_content", + "got": "expected_content", + "got_raw": "expected_content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "output", + "got": "output", + "got_raw": "output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "{\"classes\": [\"compact\", \"display\", \"nowrap\"]}", + "got": "{\"classes\": [\"compact\", \"display\", \"nowrap\"]}", + "got_raw": "{\"classes\": [\"compact\", \"display\", \"nowrap\"]}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "ValueError, match=f\"Failed to load ITables config from {tmp_path / 'itables.toml'}\")", + "got": "pytest.raises(ValueError)", + "got_raw": " pytest.raises(ValueError)", + "exact_match": false, + "code_bleu": 0.2095384381863861, + "edit_similarity": 0.2018348623853211 + }, + { + "repo": "mwouts/itables", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "size_in_kb", + "got": "size_in_kb", + "got_raw": "size_in_kb", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "{1}", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.30120066786994826, + "edit_similarity": 0.0 + }, + { + "repo": "mwouts/itables", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "set()", + "got": "set()", + "got_raw": "set()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "{2, 3}", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.18072040072196896, + "edit_similarity": 0.0 + }, + { + "repo": "mwouts/itables", + "expected": "dt_args", + "got": "dt_args", + "got_raw": "dt_args", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "0", + "got": "-1", + "got_raw": "-1", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "mwouts/itables", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "256", + "got": "256", + "got_raw": "256", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "2**16", + "got": "2**16", + "got_raw": "2**16", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "2**20", + "got": "2**20", + "got_raw": "2**20", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "\"...\"", + "got": "df.iloc[0", + "got_raw": "df.iloc[0, 0]", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "mwouts/itables", + "expected": "(5, 5)", + "got": "(10, 4)", + "got_raw": "(10, 4)", + "exact_match": false, + "code_bleu": 0.32466791547509893, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "mwouts/itables", + "expected": "len(df)", + "got": "len(df)", + "got_raw": "len(df)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "len(dn)", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.0 + }, + { + "repo": "mwouts/itables", + "expected": "(10, 5)", + "got": "(20, 5)", + "got_raw": "(20, 5)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "mwouts/itables", + "expected": "max_rows", + "got": "max_rows", + "got_raw": "max_rows", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "(10, 10)", + "got": "(10, 1.0)", + "got_raw": "(10, 1.0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.9411764705882353 + }, + { + "repo": "mwouts/itables", + "expected": "df.index)", + "got": "df.index)", + "got_raw": "df.index)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "df.columns", + "got": "df.columns", + "got_raw": "df.columns", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "{ \"classes\", \"style\", \"caption\", \"selected_rows\", }", + "got": "{\"format_floats_in_python\"}", + "got_raw": "{\"format_floats_in_python\"}", + "exact_match": false, + "code_bleu": 0.14999999999999997, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "mwouts/itables", + "expected": "{ \"table_html\", \"data_json\", \"column_filters\", \"layout\", \"order\", \"text_in_header_can_be_selected\", \"filtered_row_count\", \"downsampling_warning\", }", + "got": "set(df.columns)", + "got_raw": "set(df.columns)", + "exact_match": false, + "code_bleu": 0.061916859997111995, + "edit_similarity": 0.09876543209876543 + }, + { + "repo": "mwouts/itables", + "expected": "allow_html", + "got": "allow_html", + "got_raw": "allow_html", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "html", + "got": "html", + "got_raw": "html", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "first_row", + "got": "first_row", + "got_raw": "first_row", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "len(df) <= 10", + "got": "True", + "got_raw": "True, None)", + "exact_match": false, + "code_bleu": 0.07142857142857142, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "mwouts/itables", + "expected": "expected_tfoot", + "got": "expected_tfoot", + "got_raw": "expected_tfoot", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "len(df) <= min_rows", + "got": "len(df) <= min_rows)", + "got_raw": "len(df) <= min_rows)", + "exact_match": false, + "code_bleu": 0.8633400213704505, + "edit_similarity": 0.9743589743589743 + }, + { + "repo": "mwouts/itables", + "expected": "\"line1\\nnew line2\\nline3\\n\"", + "got": "\"line1\\nnew line2\\nline3\\n\"", + "got_raw": "\"line1\\nnew line2\\nline3\\n\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "\"line1\\nnew line2\\nnew line2\\n\"", + "got": "\"line1\\nnew line2\\nnew line2\\n\"", + "got_raw": "\"line1\\nnew line2\\nnew line2\\n\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "style.replace(\" \", \"\").removesuffix(\";\")", + "got": "style", + "got_raw": "style", + "exact_match": false, + "code_bleu": 0.04573873519241235, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "mwouts/itables", + "expected": "ValueError, match=\"cannot start with a number\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.3793103448275862 + }, + { + "repo": "mwouts/itables", + "expected": "ValueError, match=\"was found 0 times in template\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.36065573770491804 + }, + { + "repo": "mwouts/itables", + "expected": "ValueError, match=\"was found 2 times in template, expected 1.\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.2972972972972973 + }, + { + "repo": "mwouts/itables", + "expected": "f\"https://www.unpkg.com/dt_for_itables@{bundle_version}/dt_bundle.js\"", + "got": "bundle_version", + "got_raw": "bundle_version", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.3373493975903614 + }, + { + "repo": "mwouts/itables", + "expected": "f\"https://www.unpkg.com/dt_for_itables@{bundle_version}/dt_bundle.css\"", + "got": "UNPKG_DT_BUNDLE_CSS", + "got_raw": "UNPKG_DT_BUNDLE_CSS", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.06741573033707865 + }, + { + "repo": "mwouts/itables", + "expected": "{ \"classes\": get_compact_classes(opt.classes), \"style\": get_compact_style(opt.style), \"data_json\": \"[[1], [2]]\", \"order\": [], \"display_logo_when_loading\": opt.display_logo_when_loading, \"warn_on_undocumented_option\": opt.warn_on_undocumented_option, \"warn_on_unexpected_option_type\": opt.warn_on_unexpected_option_type, \"dt_url\": opt.dt_url, \"layout\": {k: None for k in opt.layout}, \"text_in_header_can_be_selected\": True, }", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.01674939598595382, + "edit_similarity": 0.009389671361502348 + }, + { + "repo": "mwouts/itables", + "expected": "{ \"classes\": get_compact_classes(opt.classes), \"style\": get_compact_style(opt.style), \"data_json\": '[[\"a\", 1], [\"b\", 2]]', \"order\": [], \"display_logo_when_loading\": opt.display_logo_when_loading, \"warn_on_undocumented_option\": opt.warn_on_undocumented_option, \"warn_on_unexpected_option_type\": opt.warn_on_unexpected_option_type, \"dt_url\": opt.dt_url, \"layout\": {k: None for k in opt.layout}, \"text_in_header_can_be_selected\": True, }", + "got": "{ \"table_id\": \"index\", \"table_html\": None, \"connected\": False, }", + "got_raw": "{ \"table_id\": \"index\", \"table_html\": None, \"connected\": False, }", + "exact_match": false, + "code_bleu": 0.03430157228999877, + "edit_similarity": 0.04417670682730924 + }, + { + "repo": "mwouts/itables", + "expected": "[[\"columnDefs\", 0, \"createdCell\"]]", + "got": "[\"columnDefs\"]", + "got_raw": "[\"columnDefs\"]", + "exact_match": false, + "code_bleu": 0.21298103474875746, + "edit_similarity": 0.5833333333333334 + }, + { + "repo": "mwouts/itables", + "expected": "0", + "got": "MARIMO_APPS_PATH.exists())", + "got_raw": "MARIMO_APPS_PATH.exists())", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.0 + }, + { + "repo": "mwouts/itables", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "dr_pd", + "got": "dr_pd", + "got_raw": "dr_pd", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "it_args", + "got": "it_args", + "got_raw": "it_args", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "it_args_nw", + "got": "it_args_nw", + "got_raw": "it_args_nw", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "dt_args", + "got": "dt_args", + "got_raw": "dt_args", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "table_html", + "got": "table_html", + "got_raw": "table_html", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "showIndex is True", + "got": "showIndex", + "got_raw": "showIndex", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.6923076923076923 + }, + { + "repo": "mwouts/itables", + "expected": "dt_args[\"layout\"].values()", + "got": "dt_args[\"buttons\"]", + "got_raw": "dt_args[\"buttons\"]", + "exact_match": false, + "code_bleu": 0.25406637407730737, + "edit_similarity": 0.5909090909090909 + }, + { + "repo": "mwouts/itables", + "expected": "5", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "mwouts/itables", + "expected": "30", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "mwouts/itables", + "expected": "100", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "mwouts/itables", + "expected": "7e9", + "got": "100", + "got_raw": "100", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "mwouts/itables", + "expected": "500", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "mwouts/itables", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "rows", + "got": "rows", + "got_raw": "rows", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "cols", + "got": "cols", + "got_raw": "cols", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "5", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "mwouts/itables", + "expected": "30", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "mwouts/itables", + "expected": "100", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "mwouts/itables", + "expected": "7e9", + "got": "100", + "got_raw": "100", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "mwouts/itables", + "expected": "500", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "mwouts/itables", + "expected": "dt_args", + "got": "dt_args", + "got_raw": "dt_args", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "itable_args", + "got": "itable_args", + "got_raw": "itable_args", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "\"[[1, 4], [2, 5], [3, 6]]\"", + "got": "df.to_json()", + "got_raw": "df.to_json()", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.0 + }, + { + "repo": "mwouts/itables", + "expected": "[pl.Int64, pl.Float64, pl.String]", + "got": "pl.DataFrame({\"A\": pl.Float64, \"B\": pl.Float64, \"C\": pl.Utf8})", + "got_raw": "pl.DataFrame({\"A\": pl.Float64, \"B\": pl.Float64, \"C\": pl.Utf8})", + "exact_match": false, + "code_bleu": 0.18556675930109115, + "edit_similarity": 0.4421052631578947 + }, + { + "repo": "mwouts/itables", + "expected": "[pl.Categorical, pl.Enum, pl.Int64]", + "got": "(pl.Categorical, cardinal_directions, pl.Int64)", + "got_raw": "(pl.Categorical, cardinal_directions, pl.Int64)", + "exact_match": false, + "code_bleu": 0.4493690680590438, + "edit_similarity": 0.6585365853658537 + }, + { + "repo": "mwouts/itables", + "expected": "'[[\"{\\\\\"C\\\\\",3}\"], [\"{\\\\\"A\\\\\",2}\"], [\"{\\\\\"B\\\\\",1}\"]]'", + "got": "{\"X\": \"A\"}", + "got_raw": "{\"X\": \"A\"}", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.25396825396825395 + }, + { + "repo": "mwouts/itables", + "expected": "'[[\"a\", \"north\", 1], [\"b\", \"south\", 2], [\"a\", \"north\", 1], [\"c\", \"west\", 3]]'", + "got": "\"columns\" in dt_args", + "got_raw": "\"columns\" in dt_args", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.10309278350515463 + }, + { + "repo": "mwouts/itables", + "expected": "\"[[1666767918216000000, 1699300000000, 0.9510565400123596, -0.30901700258255005]]\"", + "got": "\"bigint: 1666767918216000000, int: 1699300000000, float: 0.9510565400123596, neg: -0.30901700258255005\"", + "got_raw": "\"bigint: 1666767918216000000, int: 1699300000000, float: 0.9510565400123596, neg: -0.30901700258255005\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8432432432432433 + }, + { + "repo": "mwouts/itables", + "expected": "\"[[1666767918216000000, 1699300000000, \" '[\"0.951057\", 1], ' '[\"-0.309017\", 1]]]'", + "got": "\"[[1666767918216000000, 1699300000000, 0.9510565400123596, -0.30901700258255005]]\"", + "got_raw": "\"[[1666767918216000000, 1699300000000, 0.9510565400123596, -0.30901700258255005]]\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.7239263803680982 + }, + { + "repo": "mwouts/itables", + "expected": "'[[1, 0.1, \"x\"], ' \"[2, null, null], \" '[null, \"___NaN___\", \"z\"], ' '[4, 0.4, \"w\"]]'", + "got": "dt_args[\"data_json\"]", + "got_raw": "dt_args[\"data_json\"]", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.07692307692307693 + }, + { + "repo": "mwouts/itables", + "expected": "'[[1, [\"0.1\", 1], \"x\"], ' \"[2, [null, 0], null], \" '[null, [\"NaN\", 3], \"z\"], ' '[4, [\"0.4\", 2], \"w\"]]'", + "got": "dt_args[\"data_json\"]", + "got_raw": "dt_args[\"data_json\"]", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.06557377049180328 + }, + { + "repo": "mwouts/itables", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "0", + "got": "SHINY_APPS_PY_FILES", + "got_raw": "SHINY_APPS_PY_FILES", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "mwouts/itables", + "expected": "IndexError, match=\"Selected rows out of range\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.17909498863725698, + "edit_similarity": 0.2413793103448276 + }, + { + "repo": "mwouts/itables", + "expected": "captured.out", + "got": "captured", + "got_raw": "captured", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.8 + }, + { + "repo": "mwouts/itables", + "expected": "0", + "got": "len(STREAMLIT_APPS_PY_FILES)", + "got_raw": "len(STREAMLIT_APPS_PY_FILES)", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.0 + }, + { + "repo": "mwouts/itables", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "df", + "got": "df", + "got_raw": "df", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mwouts/itables", + "expected": "opt.style", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.0 + }, + { + "repo": "mwouts/itables", + "expected": "opt.classes", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.30120066786994826, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "2", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "0", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "4", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "2", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "8", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "2", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "5", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "3", + "got": "4", + "got_raw": "4", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "0", + "got": "event_queue.qsize()", + "got_raw": "event_queue.qsize()", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "33 - 7", + "got": "33", + "got_raw": "33", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5 + }, + { + "repo": "nolar/kopf", + "expected": "33 - 5", + "got": "33 - 5", + "got_raw": "33 - 5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "123 * 2", + "got": "e.value.delay", + "got_raw": "e.value.delay", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "proxy.url", + "got": "proxy.url", + "got_raw": "proxy.url", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "cause.new", + "got": "cause.new", + "got_raw": "cause.new", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "cause.old", + "got": "cause.old", + "got_raw": "cause.old", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "cause.memo", + "got": "cause.memo", + "got_raw": "cause.memo", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "cause.body", + "got": "cause.body", + "got_raw": "cause.body", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "cause.diff", + "got": "cause.diff", + "got_raw": "cause.diff", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "cause.type", + "got": "cause.type", + "got_raw": "cause.type", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "view2._src", + "got": "view2", + "got_raw": "view2", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "nolar/kopf", + "expected": "certpkey[0]", + "got": "certpkey", + "got_raw": "certpkey", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.8421052631578947 + }, + { + "repo": "nolar/kopf", + "expected": "Reason.GONE", + "got": "Reason.GONE", + "got_raw": "Reason.GONE", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "Reason.FREE", + "got": "Reason.NONE", + "got_raw": "Reason.NONE", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "nolar/kopf", + "expected": "Reason.NOOP", + "got": "Reason.NO_OP", + "got_raw": "Reason.NO_OP", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.9565217391304348 + }, + { + "repo": "nolar/kopf", + "expected": "body['spec']", + "got": "body['spec']", + "got_raw": "body['spec']", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "d", + "got": "d", + "got_raw": "d", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "d", + "got": "d", + "got_raw": "d", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "fn", + "got": "fn", + "got_raw": "fn", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "id", + "got": "id", + "got_raw": "id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "d2", + "got": "d2", + "got_raw": "d2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "fn", + "got": "fn", + "got_raw": "fn", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "fn", + "got": "fn", + "got_raw": "fn", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "url", + "got": "url", + "got_raw": "url", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "exc", + "got": "exc", + "got_raw": "exc", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "old", + "got": "old", + "got_raw": "old", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "new", + "got": "new", + "got_raw": "new", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "old", + "got": "old", + "got_raw": "old", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "new", + "got": "new", + "got_raw": "new", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "obj", + "got": "obj", + "got_raw": "obj", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "obj", + "got": "obj", + "got_raw": "obj", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "ts0", + "got": "ts0", + "got_raw": "ts0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "obj", + "got": "obj", + "got_raw": "obj", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "obj", + "got": "obj", + "got_raw": "obj", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "uid", + "got": "uid", + "got_raw": "uid", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "code", + "got": "code", + "got_raw": "code", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "info", + "got": "info", + "got_raw": "info", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "key1", + "got": "key1", + "got_raw": "key1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "key2", + "got": "key2", + "got_raw": "key2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "obj1", + "got": "obj1", + "got_raw": "obj1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "memo", + "got": "memo", + "got_raw": "memo", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "body", + "got": "body", + "got_raw": "body", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "type", + "got": "type", + "got_raw": "type", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "diff", + "got": "diff", + "got_raw": "diff", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "when", + "got": "when", + "got_raw": "when", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "obj1", + "got": "obj1", + "got_raw": "obj1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "obj2", + "got": "obj2", + "got_raw": "obj2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "DIFF", + "got": "DIFF", + "got_raw": "DIFF", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "obj1", + "got": "obj1", + "got_raw": "obj1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "obj2", + "got": "obj2", + "got_raw": "obj2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "keys", + "got": "keys", + "got_raw": "keys", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "body", + "got": "body", + "got_raw": "body", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "REF1", + "got": "REF1", + "got_raw": "REF1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "REF1", + "got": "REF1", + "got_raw": "REF1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "REF2", + "got": "REF2", + "got_raw": "REF2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "cert1", + "got": "cert1", + "got_raw": "cert1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "cert2", + "got": "cert2", + "got_raw": "cert2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "True", + "got": "False", + "got_raw": "False", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "nolar/kopf", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "True", + "got": "False", + "got_raw": "False", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "nolar/kopf", + "expected": "{}", + "got": "{'metadata':", + "got_raw": "{'metadata': {'finalizers': ['other1', 'other2']}}", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "nolar/kopf", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "()", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "()", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "[]", + "got": "[call(123)]", + "got_raw": "[call(123)]", + "exact_match": false, + "code_bleu": 0.24446151121745047, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "nolar/kopf", + "expected": "{1}", + "got": "{1}", + "got_raw": "{1}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "{r1}", + "got": "fn", + "got_raw": "fn", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "{r1}", + "got": "{", + "got_raw": "{ r1: EVERYTHING }", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.4 + }, + { + "repo": "nolar/kopf", + "expected": "{r2}", + "got": "{", + "got_raw": "{ r1: EVERYTHING, r2: EVERYTHING }", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.4 + }, + { + "repo": "nolar/kopf", + "expected": "{''}", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.30120066786994826, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "[5.0]", + "got": "[5.0", + "got_raw": "[5.0, 5.0]", + "exact_match": false, + "code_bleu": 0.4714045207910317, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "nolar/kopf", + "expected": "{123}", + "got": "{key}", + "got_raw": "{key}", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.4 + }, + { + "repo": "nolar/kopf", + "expected": "{123}", + "got": "{memory}", + "got_raw": "{memory}", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "nolar/kopf", + "expected": "{123}", + "got": "{'name1'}", + "got_raw": "{'name1'}", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.42857142857142855 + }, + { + "repo": "nolar/kopf", + "expected": "{456}", + "got": "{456}", + "got_raw": "{456}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "[123]", + "got": "[123]", + "got_raw": "[123]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "{None}", + "got": "{key}", + "got_raw": "{key}", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "nolar/kopf", + "expected": "{None}", + "got": "set()", + "got_raw": "set()", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "nolar/kopf", + "expected": "{None}", + "got": "{'name1'}", + "got_raw": "{'name1'}", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "nolar/kopf", + "expected": "{task}", + "got": "[task]", + "got_raw": "[task]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "nolar/kopf", + "expected": "{'xyz'}", + "got": "{'xyz'}", + "got_raw": "{'xyz'}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "{'ns1'}", + "got": "['ns1']", + "got_raw": "['ns1']", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "nolar/kopf", + "expected": "{'ns1'}", + "got": "['ns1']", + "got_raw": "['ns1']", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "nolar/kopf", + "expected": "{r1ns1}", + "got": "{r1ns1", + "got_raw": "{r1ns1, r1ns2, r2ns1, r2ns2}", + "exact_match": false, + "code_bleu": 0.4714045207910317, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "nolar/kopf", + "expected": "{peer1}", + "got": "{peer1}", + "got_raw": "{peer1}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "{peerN}", + "got": "{peerN}", + "got_raw": "{peerN}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "{'id1'}", + "got": "{'status':", + "got_raw": "{'status': 'purged'}", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "nolar/kopf", + "expected": "[{}, {}]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.16988673071467442, + "edit_similarity": 0.4 + }, + { + "repo": "nolar/kopf", + "expected": "{r1, r2}", + "got": "{'group1': r1, 'group2': r2}", + "got_raw": "{'group1': r1, 'group2': r2}", + "exact_match": false, + "code_bleu": 0.24446151121745052, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "nolar/kopf", + "expected": "['oops!']", + "got": "[\"oops!\"]", + "got_raw": "[\"oops!\"]", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "nolar/kopf", + "expected": "['field']", + "got": "['field']", + "got_raw": "['field']", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "['value']", + "got": "['value']", + "got_raw": "['value']", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "\"\"", + "got": "\"APIError(status=456", + "got_raw": "\"APIError(status=456, code=None, message=None)\"", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.09090909090909091 + }, + { + "repo": "nolar/kopf", + "expected": "''", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "''", + "got": "'v1'", + "got_raw": "'v1'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "nolar/kopf", + "expected": "'a'", + "got": "'a'", + "got_raw": "'a'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'b'", + "got": "'b'", + "got_raw": "'b'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'a'", + "got": "'a'", + "got_raw": "'a'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'b'", + "got": "'b'", + "got_raw": "'b'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'c'", + "got": "'c'", + "got_raw": "'c'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'d'", + "got": "'d'", + "got_raw": "'d'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'a'", + "got": "'a'", + "got_raw": "'a'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'b'", + "got": "'b'", + "got_raw": "'b'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'a'", + "got": "'a'", + "got_raw": "'a'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'b'", + "got": "'b'", + "got_raw": "'b'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'x'", + "got": "'x'", + "got_raw": "'x'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'1'", + "got": "'3'", + "got_raw": "'3'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "nolar/kopf", + "expected": "'2'", + "got": "'2'", + "got_raw": "'2'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'a'", + "got": "'a'", + "got_raw": "'a'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'xx'", + "got": "'xx'", + "got_raw": "'xx'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'ns'", + "got": "'ns'", + "got_raw": "'ns'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'ns'", + "got": "'ns'", + "got_raw": "'ns'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'id'", + "got": "'id'", + "got_raw": "'id'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'\\n'", + "got": "''", + "got_raw": "''", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "nolar/kopf", + "expected": "'xyz'", + "got": "'xyz'", + "got_raw": "'xyz'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'xyz'", + "got": "'xyz'", + "got_raw": "'xyz'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'xyz'", + "got": "'xyz'", + "got_raw": "'xyz'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "'xyz'", + "got": "'xyz'", + "got_raw": "'xyz'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "\"boo\"", + "got": "\"Error", + "got_raw": "\"Error while fetching '/url': Error while fetching '/url': Error while fetching '/url': Error while fetching '/url': Error while fetching '/url': Error while fetching '/url': Error while fetching '/url': Error while fetching '/url': Error while fetching '/url': Error while fetching '/url': Error while fetching '/url': Error while fetching '/url': Error while fetching '/url': Error while fetching '/url': Error while fetching '/url': Error while fetching '/url': Error while fetching '/url': Error while fetching '/url': Error", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "nolar/kopf", + "expected": "'tkn'", + "got": "'tkn'", + "got_raw": "'tkn'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "set()", + "got": "set(resource_indexed)", + "got_raw": "set(resource_indexed)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.38461538461538464 + }, + { + "repo": "nolar/kopf", + "expected": "set()", + "got": "{'name1'}", + "got_raw": "{'name1'}", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "nolar/kopf", + "expected": "set()", + "got": "{456}", + "got_raw": "{456}", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "set()", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.30120066786994826, + "edit_similarity": 0.0 + }, + { + "repo": "nolar/kopf", + "expected": "set()", + "got": "set()", + "got_raw": "set()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nolar/kopf", + "expected": "hash(d2)", + "got": "hash(d2)", + "got_raw": "hash(d2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "d)", + "got": "d)", + "got_raw": "d)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "n)", + "got": "n)", + "got_raw": "n)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a)", + "got": "a)", + "got_raw": "a)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "b)", + "got": "b)", + "got_raw": "b)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "c)", + "got": "c)", + "got_raw": "c)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a)", + "got": "a)", + "got_raw": "a)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a)", + "got": "a)", + "got_raw": "a)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "b)", + "got": "b)", + "got_raw": "b)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "c)", + "got": "c)", + "got_raw": "c)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "d)", + "got": "d)", + "got_raw": "d)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a)", + "got": "a)", + "got_raw": "a)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a)", + "got": "a)", + "got_raw": "a)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a)", + "got": "b)", + "got_raw": "b)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "octodns/octodns", + "expected": "b)", + "got": "b)", + "got_raw": "b)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a)", + "got": "a)", + "got_raw": "a)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "b)", + "got": "b)", + "got_raw": "b)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "c)", + "got": "c)", + "got_raw": "c)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a)", + "got": "a)", + "got_raw": "a)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "b)", + "got": "b)", + "got_raw": "b)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "c)", + "got": "c)", + "got_raw": "c)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a)", + "got": "a)", + "got_raw": "a)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "b)", + "got": "b)", + "got_raw": "b)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "c)", + "got": "c)", + "got_raw": "c)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a)", + "got": "a)", + "got_raw": "a)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a)", + "got": "a)", + "got_raw": "a)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "b)", + "got": "b)", + "got_raw": "b)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "c)", + "got": "c)", + "got_raw": "c)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "d)", + "got": "d)", + "got_raw": "d)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "e)", + "got": "e)", + "got_raw": "e)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a)", + "got": "a)", + "got_raw": "a)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "b)", + "got": "b)", + "got_raw": "b)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "c)", + "got": "c)", + "got_raw": "c)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "d)", + "got": "d)", + "got_raw": "d)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "b)", + "got": "b)", + "got_raw": "b)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a)", + "got": "a)", + "got_raw": "a)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "c)", + "got": "c)", + "got_raw": "c)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "d)", + "got": "d)", + "got_raw": "d)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "e)", + "got": "e)", + "got_raw": "e)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a)", + "got": "a)", + "got_raw": "a)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a)", + "got": "a)", + "got_raw": "a)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "b)", + "got": "b)", + "got_raw": "b)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "c)", + "got": "c)", + "got_raw": "c)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "e)", + "got": "e)", + "got_raw": "e)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a)", + "got": "a)", + "got_raw": "a)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "v)", + "got": "v)", + "got_raw": "v)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "b)", + "got": "b)", + "got_raw": "b)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "sh)", + "got": "sh)", + "got_raw": "sh)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "tc)", + "got": "len(tc))", + "got_raw": "len(tc))", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "octodns/octodns", + "expected": "one)", + "got": "same)", + "got_raw": "same)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "octodns/octodns", + "expected": "got)", + "got": "got)", + "got_raw": "got)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got)", + "got": "got)", + "got_raw": "got)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "out)", + "got": "out)", + "got_raw": "out)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "ttl)", + "got": "ttl)", + "got_raw": "ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got)", + "got": "got)", + "got_raw": "got)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got)", + "got": "got)", + "got_raw": "got)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got)", + "got": "got)", + "got_raw": "got)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "one)", + "got": "one)", + "got_raw": "one)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "two)", + "got": "two)", + "got_raw": "two)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "msg)", + "got": "msg)", + "got_raw": "msg)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got)", + "got": "got)", + "got_raw": "got)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got)", + "got": "got)", + "got_raw": "got)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got)", + "got": "got)", + "got_raw": "got)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got)", + "got": "got)", + "got_raw": "got)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got)", + "got": "got)", + "got_raw": "got)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got)", + "got": "got)", + "got_raw": "got)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got)", + "got": "got)", + "got_raw": "got)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got)", + "got": "got)", + "got_raw": "got)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got)", + "got": "got)", + "got_raw": "got)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got)", + "got": "got)", + "got_raw": "got)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got)", + "got": "got)", + "got_raw": "got)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got)", + "got": "got)", + "got_raw": "got)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got)", + "got": "got)", + "got_raw": "got)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "str)", + "got": "str)", + "got_raw": "str)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "int)", + "got": "int)", + "got_raw": "int)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "same)", + "got": "same)", + "got_raw": "same)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "data)", + "got": "data)", + "got_raw": "data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "list)", + "got": "list)", + "got_raw": "list)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "plan)", + "got": "plan)", + "got_raw": "plan)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "plan)", + "got": "plan)", + "got_raw": "plan)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "plan)", + "got": "plan)", + "got_raw": "plan)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "data)", + "got": "data)", + "got_raw": "data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "value)", + "got": "value)", + "got_raw": "value)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "found)", + "got": "found)", + "got_raw": "found)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "other)", + "got": "other)", + "got_raw": "other)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "len(d))", + "got": "len(d))", + "got_raw": "len(d))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "set([a]))", + "got": "[a])", + "got_raw": "[a])", + "exact_match": false, + "code_bleu": 0.5714285714285714, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "octodns/octodns", + "expected": "len(data))", + "got": "len(data))", + "got_raw": "len(data))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "len(values))", + "got": "len(values))", + "got_raw": "len(values))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "set([a, b]))", + "got": "set([a, b]))", + "got_raw": "set([a, b]))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "set([a, c]))", + "got": "[c, a])", + "got_raw": "[c, a])", + "exact_match": false, + "code_bleu": 0.23953607464205617, + "edit_similarity": 0.5263157894736842 + }, + { + "repo": "octodns/octodns", + "expected": "len(changes))", + "got": "len(changes))", + "got_raw": "len(changes))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "buf.getvalue())", + "got": "buf.getvalue())", + "got_raw": "buf.getvalue())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "tuple(d.keys()))", + "got": "set(self.normal.keys()))", + "got_raw": "set(self.normal.keys()))", + "exact_match": false, + "code_bleu": 0.5331675363405771, + "edit_similarity": 0.55 + }, + { + "repo": "octodns/octodns", + "expected": "len(txt.values))", + "got": "len(txt.values))", + "got_raw": "len(txt.values))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "sorted(changes))", + "got": "changes)", + "got_raw": "changes)", + "exact_match": false, + "code_bleu": 0.28284271247461906, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "octodns/octodns", + "expected": "idna_decode(got))", + "got": "idna_decode(got))", + "got_raw": "idna_decode(got))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "tuple(d.items()))", + "got": "(self.normal.items())", + "got_raw": "(self.normal.items())", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5263157894736842 + }, + { + "repo": "octodns/octodns", + "expected": "len(got.records))", + "got": "len(got.records))", + "got_raw": "len(got.records))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "len(zone.records))", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.153509990260506, + "edit_similarity": 0.1 + }, + { + "repo": "octodns/octodns", + "expected": "len(zone.records))", + "got": "len(zone.records))", + "got_raw": "len(zone.records))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "len(plan.changes))", + "got": "len(plan.changes))", + "got_raw": "len(plan.changes))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "len(zone.records))", + "got": "len(zone.records))", + "got_raw": "len(zone.records))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "str(ctx.exception))", + "got": "'Invalid", + "got_raw": "'Invalid IDNA domain name')", + "exact_match": false, + "code_bleu": 0.1290860005156921, + "edit_similarity": 0.07407407407407407 + }, + { + "repo": "octodns/octodns", + "expected": "str(ctx.exception))", + "got": "'1.2.3.4'", + "got_raw": "'1.2.3.4', ctx.exception.value)", + "exact_match": false, + "code_bleu": 0.07142857142857142, + "edit_similarity": 0.07142857142857142 + }, + { + "repo": "octodns/octodns", + "expected": "str(ctx.exception))", + "got": "'bad'", + "got_raw": "'bad'", + "exact_match": false, + "code_bleu": 0.07142857142857142, + "edit_similarity": 0.0 + }, + { + "repo": "octodns/octodns", + "expected": "value.template({}))", + "got": "value)", + "got_raw": "value)", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.48 + }, + { + "repo": "octodns/octodns", + "expected": "len(single.values))", + "got": "len(single.values))", + "got_raw": "len(single.values))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "str(ctx.exception))", + "got": "ctx.exception.message)", + "got_raw": "ctx.exception.message)", + "exact_match": false, + "code_bleu": 0.3726178902385641, + "edit_similarity": 0.6829268292682927 + }, + { + "repo": "octodns/octodns", + "expected": "str(ctx.exception))", + "got": "'include-doesnt-exist.yaml'", + "got_raw": "'include-doesnt-exist.yaml' not found", + "exact_match": false, + "code_bleu": 0.07142857142857142, + "edit_similarity": 0.21739130434782608 + }, + { + "repo": "octodns/octodns", + "expected": "GeoCodes.parse('NA'))", + "got": "GeoCodes('US-CA-12345-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345", + "got_raw": "GeoCodes('US-CA-12345-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345", + "exact_match": false, + "code_bleu": 0.13950796967929133, + "edit_similarity": 0.13924050632911392 + }, + { + "repo": "octodns/octodns", + "expected": "a.changes(a, target))", + "got": "a.changes(target))", + "got_raw": "a.changes(target))", + "exact_match": false, + "code_bleu": 0.5377878872207972, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "octodns/octodns", + "expected": "a.changes(b, target))", + "got": "b.changes(a, target))", + "got_raw": "b.changes(a, target))", + "exact_match": false, + "code_bleu": 0.5555238068023581, + "edit_similarity": 0.9047619047619048 + }, + { + "repo": "octodns/octodns", + "expected": "a.changes(c, target))", + "got": "a.changes(c, target))", + "got_raw": "a.changes(c, target))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "len(filtered.records))", + "got": "len(filtered))", + "got_raw": "len(filtered))", + "exact_match": false, + "code_bleu": 0.4316250567652526, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "octodns/octodns", + "expected": "len(with_dot.records))", + "got": "len(with_dot.records))", + "got_raw": "len(with_dot.records))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "str(context.exception))", + "got": "'Invalid", + "got_raw": "'Invalid TXT value: v=spf1 ptr ~all')", + "exact_match": false, + "code_bleu": 0.1290860005156921, + "edit_similarity": 0.12903225806451613 + }, + { + "repo": "octodns/octodns", + "expected": "tuple(d.decoded_keys()))", + "got": "(self.plain", + "got_raw": "(self.plain, self.almost, self.utf8))", + "exact_match": false, + "code_bleu": 0.17672682861490013, + "edit_similarity": 0.11428571428571428 + }, + { + "repo": "octodns/octodns", + "expected": "_find(got, 'has').value)", + "got": "has.value)", + "got_raw": "has.value)", + "exact_match": false, + "code_bleu": 0.2972179133228542, + "edit_similarity": 0.5882352941176471 + }, + { + "repo": "octodns/octodns", + "expected": "_find(got, 'sub').value)", + "got": "got[0].value)", + "got_raw": "got[0].value)", + "exact_match": false, + "code_bleu": 0.2843207288868429, + "edit_similarity": 0.5405405405405406 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a.ttl)", + "got_raw": "a.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a.ttl)", + "got_raw": "a.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a.ttl)", + "got_raw": "a.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a.ttl)", + "got_raw": "a.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a.ttl)", + "got_raw": "a.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a.ttl)", + "got_raw": "a.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a < b)", + "got": "a != b)", + "got_raw": "a != b)", + "exact_match": false, + "code_bleu": 0.3860973950960897, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a._ttl)", + "got_raw": "a._ttl)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a.ttl)", + "got_raw": "a.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a.ttl)", + "got_raw": "a.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a.ttl)", + "got_raw": "a.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a.ttl)", + "got_raw": "a.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a.ttl)", + "got_raw": "a.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a.ttl)", + "got_raw": "a.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a.ttl)", + "got_raw": "a.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a.ttl)", + "got_raw": "a.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a.ttl)", + "got_raw": "a.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "b.ttl)", + "got": "b.ttl)", + "got_raw": "b.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a.ttl)", + "got_raw": "a.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a.ttl)", + "got_raw": "a.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.ttl)", + "got": "a.ttl)", + "got_raw": "a.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "b.data)", + "got": "b.data)", + "got_raw": "b.data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "c.data)", + "got": "c.data)", + "got_raw": "c.data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "b.name)", + "got": "b.name)", + "got_raw": "b.name)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "c.name)", + "got": "c.name)", + "got_raw": "c.name)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.name)", + "got": "a.name)", + "got_raw": "a.name)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.fqdn)", + "got": "a.fqdn)", + "got_raw": "a.fqdn)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.data)", + "got": "a.to_dict())", + "got_raw": "a.to_dict())", + "exact_match": false, + "code_bleu": 0.31239399369202553, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "octodns/octodns", + "expected": "b.data)", + "got": "b.data)", + "got_raw": "b.data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.name)", + "got": "a.name)", + "got_raw": "a.name)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.fqdn)", + "got": "a.fqdn)", + "got_raw": "a.fqdn)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.data)", + "got": "a.to_data())", + "got_raw": "a.to_data())", + "exact_match": false, + "code_bleu": 0.31239399369202553, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "octodns/octodns", + "expected": "b.data)", + "got": "b.data)", + "got_raw": "b.data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.name)", + "got": "a.value)", + "got_raw": "a.value)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "octodns/octodns", + "expected": "a.fqdn)", + "got": "a.zone.name)", + "got_raw": "a.zone.name)", + "exact_match": false, + "code_bleu": 0.31239399369202553, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "octodns/octodns", + "expected": "a.name)", + "got": "a.name)", + "got_raw": "a.name)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.fqdn)", + "got": "a.fqdn)", + "got_raw": "a.fqdn)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.name)", + "got": "a.name)", + "got_raw": "a.name)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.fqdn)", + "got": "a.fqdn)", + "got_raw": "a.fqdn)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.name)", + "got": "a.name)", + "got_raw": "a.name)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.fqdn)", + "got": "a.fqdn)", + "got_raw": "a.fqdn)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.data)", + "got": "a.to_dict())", + "got_raw": "a.to_dict())", + "exact_match": false, + "code_bleu": 0.31239399369202553, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "octodns/octodns", + "expected": "b.data)", + "got": "b.data)", + "got_raw": "b.data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.name)", + "got": "a.name)", + "got_raw": "a.name)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.fqdn)", + "got": "a.fqdn)", + "got_raw": "a.fqdn)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.data)", + "got": "a.to_dict())", + "got_raw": "a.to_dict())", + "exact_match": false, + "code_bleu": 0.31239399369202553, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "octodns/octodns", + "expected": "b.data)", + "got": "b.data)", + "got_raw": "b.data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.name)", + "got": "a.name)", + "got_raw": "a.name)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.fqdn)", + "got": "a.fqdn)", + "got_raw": "a.fqdn)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.data)", + "got": "a.to_data())", + "got_raw": "a.to_data())", + "exact_match": false, + "code_bleu": 0.31239399369202553, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "octodns/octodns", + "expected": "b.data)", + "got": "b.data)", + "got_raw": "b.data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.name)", + "got": "a.name)", + "got_raw": "a.name)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.fqdn)", + "got": "a.fqdn)", + "got_raw": "a.fqdn)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a.data)", + "got": "a.to_data())", + "got_raw": "a.to_data())", + "exact_match": false, + "code_bleu": 0.31239399369202553, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "octodns/octodns", + "expected": "b.data)", + "got": "b.data)", + "got_raw": "b.data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "ptr.ttl)", + "got": "ptr.ttl)", + "got_raw": "ptr.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "a._type)", + "got": "a.type)", + "got_raw": "a.type)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "octodns/octodns", + "expected": "ptr.fqdn)", + "got": "ptr.name)", + "got_raw": "ptr.name)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "octodns/octodns", + "expected": "geo.data)", + "got": "geo.geo)", + "got_raw": "geo.geo)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.5882352941176471 + }, + { + "repo": "octodns/octodns", + "expected": "got.path)", + "got": "got)", + "got_raw": "got)", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "octodns/octodns", + "expected": "ptr.value)", + "got": "ptr.value)", + "got_raw": "ptr.value)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "ptr_2.ttl)", + "got": "ptr_1.ttl)", + "got_raw": "ptr_1.ttl)", + "exact_match": false, + "code_bleu": 0.668740304976422, + "edit_similarity": 0.9 + }, + { + "repo": "octodns/octodns", + "expected": "proc.uuid)", + "got": "proc.needs_update(self.meta_needs_update))", + "got_raw": "proc.needs_update(self.meta_needs_update))", + "exact_match": false, + "code_bleu": 0.19960198807747329, + "edit_similarity": 0.2692307692307692 + }, + { + "repo": "octodns/octodns", + "expected": "proc.time)", + "got": "proc.time)", + "got_raw": "proc.time)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "meta.name)", + "got": "meta.name)", + "got_raw": "meta.name)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got.value)", + "got": "got)", + "got_raw": "got)", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "octodns/octodns", + "expected": "zone.name)", + "got": "zone.name)", + "got_raw": "zone.name)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "ptr_1.fqdn)", + "got": "ptr_1.name)", + "got_raw": "ptr_1.name)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "octodns/octodns", + "expected": "ptr_2.fqdn)", + "got": "ptr_2.fqdn)", + "got_raw": "ptr_2.fqdn)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "ptr.values)", + "got": "ptr.values)", + "got_raw": "ptr.values)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "meta._type)", + "got": "meta.type)", + "got_raw": "meta.type)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.9523809523809523 + }, + { + "repo": "octodns/octodns", + "expected": "record.ttl)", + "got": "record.ttl)", + "got_raw": "record.ttl)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "noop.value)", + "got": "noop.value)", + "got_raw": "noop.value)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got.values)", + "got": "got.values)", + "got_raw": "got.values)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "got.digest)", + "got": "got)", + "got_raw": "got)", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "octodns/octodns", + "expected": "other.data)", + "got": "other.data)", + "got_raw": "other.data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "octodns/octodns", + "expected": "'TXT')", + "got": "dns.rdatatype.TXT)", + "got_raw": "dns.rdatatype.TXT)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "octodns/octodns", + "expected": "'testvalue')", + "got": "'testvalue')", + "got_raw": "'testvalue')", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "b'\\x00\\x00'", + "got": "BITCOIN_ADDRESS[:1]", + "got_raw": "BITCOIN_ADDRESS[:1]", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.0 + }, + { + "repo": "ofek/bit", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "BINARY_ADDRESS", + "got": "BITCOIN_ADDRESS", + "got_raw": "BITCOIN_ADDRESS", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7586206896551724 + }, + { + "repo": "ofek/bit", + "expected": "BITCOIN_ADDRESS", + "got": "BITCOIN_ADDRESS", + "got_raw": "BITCOIN_ADDRESS", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "BITCOIN_ADDRESS[:1]", + "got": "BITCOIN_ADDRESS[:1]", + "got_raw": "BITCOIN_ADDRESS[:1]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "MAIN_PUBKEY_HASH + PUBKEY_HASH", + "got": "BITCOIN_ADDRESS", + "got_raw": "BITCOIN_ADDRESS", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "ofek/bit", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "ofek/bit", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "point", + "got": "point", + "got_raw": "point", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "Y_CORRECT", + "got": "(X_CORRECT", + "got_raw": "(X_CORRECT, Y_CORRECT)", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.8421052631578947 + }, + { + "repo": "ofek/bit", + "expected": "Y_INCORRECT", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "ofek/bit", + "expected": "'Point(x=5, y=10)'", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "ofek/bit", + "expected": "'main'", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "ofek/bit", + "expected": "'test'", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "ofek/bit", + "expected": "PUBKEY_HASH", + "got": "PUBKEY_HASH", + "got_raw": "PUBKEY_HASH", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "PAY2SH_HASH", + "got": "PAY2SH_HASH", + "got_raw": "PAY2SH_HASH", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "ValueError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "ofek/bit", + "expected": "BITCOIN_ADDRESS", + "got": "BITCOIN_ADDRESS", + "got_raw": "BITCOIN_ADDRESS", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "addr", + "got": "addr", + "got_raw": "addr", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "hex_to_bytes(hexscript)", + "got": "hex_to_bytes(hexscript)", + "got_raw": "hex_to_bytes(hexscript)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "ofek/bit", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "ofek/bit", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "ofek/bit", + "expected": "b''", + "got": "b'\\xff\\xff\\xff\\xff'", + "got_raw": "b'\\xff\\xff\\xff\\xff'", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.2727272727272727 + }, + { + "repo": "ofek/bit", + "expected": "HEX", + "got": "BYTES_BIG", + "got_raw": "BYTES_BIG", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "ofek/bit", + "expected": "BIG_INT", + "got": "BIG_INT", + "got_raw": "BIG_INT", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "b'\\x00'", + "got": "b'\\x00'", + "got_raw": "b'\\x00'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "BYTES_BIG", + "got": "bytes.fromhex(HEX)", + "got_raw": "bytes.fromhex(HEX)", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.07407407407407407 + }, + { + "repo": "ofek/bit", + "expected": "HEX.upper()", + "got": "'TH8E2AAANDD7^AX793E7C6A30285'", + "got_raw": "'TH8E2AAANDD7^AX793E7C6A30285'", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.14634146341463414 + }, + { + "repo": "ofek/bit", + "expected": "BYTES_LITTLE", + "got": "BYTES_LITTLE", + "got_raw": "BYTES_LITTLE", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "ODD_HEX_BYTES", + "got": "ODD_HEX_BYTES", + "got_raw": "ODD_HEX_BYTES", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "Decimal('0.8')", + "got": "Decimal(0.8)", + "got_raw": "Decimal(0.8)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "ofek/bit", + "expected": "[ '4f', 'ad', 'd1', '97', '73', '28', 'c1', '1e', 'fc', '1c', '1d', '8a', '78', '1a', 'a6', 'b9', '67', '79', '84', 'd3', 'e0', 'bd', '0b', 'fc', '52', 'b9', 'f3', 'b0', '38', '85', 'a0', '0', ]", + "got": "[ b'4fadd197', b'777328c1', b'1efc1c1d8a7', b'81aa6b96', b'7798m>\\x0b\\xd0\\xbf\\xc5+\\x9f;\\x03\\x88Z\\x00' ]", + "got_raw": "[ b'4fadd197', b'777328c1', b'1efc1c1d8a7', b'81aa6b96', b'7798m>\\x0b\\xd0\\xbf\\xc5+\\x9f;\\x03\\x88Z\\x00' ]", + "exact_match": false, + "code_bleu": 0.02550665496681267, + "edit_similarity": 0.3501683501683502 + }, + { + "repo": "ofek/bit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "pk", + "got": "pk", + "got_raw": "pk", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "262", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "ofek/bit", + "expected": "188", + "got": "131.5", + "got_raw": "131.5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.25 + }, + { + "repo": "ofek/bit", + "expected": "113", + "got": "131.5", + "got_raw": "131.5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "ofek/bit", + "expected": "298", + "got": "262", + "got_raw": "262", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "ofek/bit", + "expected": "140", + "got": "162.5", + "got_raw": "162.5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.25 + }, + { + "repo": "ofek/bit", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "None", + "got": "False", + "got_raw": "False", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "ofek/bit", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "initial", + "got": "initial", + "got_raw": "initial", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "PrivateKey", + "got": "PrivateKey", + "got_raw": "PrivateKey", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "30", + "got": "30", + "got_raw": "30", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "updated", + "got": "updated", + "got_raw": "updated", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "cached_time", + "got": "cached_time", + "got_raw": "cached_time", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "get_fee(fast=False)", + "got": "0.0001", + "got_raw": "0.0001", + "exact_match": false, + "code_bleu": 0.08333333333333333, + "edit_similarity": 0.0 + }, + { + "repo": "ofek/bit", + "expected": "7", + "got": "7", + "got_raw": "7", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "91", + "got": "128", + "got_raw": "128", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "ofek/bit", + "expected": "68", + "got": "140", + "got_raw": "140", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "ofek/bit", + "expected": "180", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "ofek/bit", + "expected": "148", + "got": "120", + "got_raw": "120", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "ofek/bit", + "expected": "298", + "got": "148", + "got_raw": "148", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "ofek/bit", + "expected": "10000", + "got": "10000", + "got_raw": "10000", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "'txid'", + "got": "'txid'", + "got_raw": "'txid'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "'script'", + "got": "'script'", + "got_raw": "'script'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "unspent2", + "got": "unspent2", + "got_raw": "unspent2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "unspent3", + "got": "unspent3", + "got_raw": "unspent3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "Unspent.from_dict(unspent.to_dict())", + "got": "dict(txid='txid'", + "got_raw": "dict(txid='txid', vout=0, scriptPubKey='script', amount=10000)", + "exact_match": false, + "code_bleu": 0.13650604313545334, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "ofek/bit", + "expected": "\"Unspent(amount=10000, confirmations=7, \" \"script='script', txid='txid', txindex=0, \" \"segwit=False, \" \"sequence=4294967295)\"", + "got": "\"Unspent(10000, 7, 'script', 'txid', 0)\")", + "got_raw": "\"Unspent(10000, 7, 'script', 'txid', 0)\")", + "exact_match": false, + "code_bleu": 0.2259005009024612, + "edit_similarity": 0.4819277108433735 + }, + { + "repo": "ofek/bit", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "0", + "got": "-2", + "got_raw": "-2", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "ofek/bit", + "expected": "30", + "got": "30", + "got_raw": "30", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "100", + "got": "100000000", + "got_raw": "100000000", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "ofek/bit", + "expected": "2000", + "got": "0.00000000000001", + "got_raw": "0.00000000000001", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3 + }, + { + "repo": "ofek/bit", + "expected": "100000", + "got": "1000000", + "got_raw": "1000000", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "ofek/bit", + "expected": "'0.01'", + "got": "0.00000001", + "got_raw": "0.00000001", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "ofek/bit", + "expected": "updated", + "got": "updated", + "got_raw": "updated", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "100000000", + "got": "100000000", + "got_raw": "100000000", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "cached_time", + "got": "cached_time", + "got_raw": "cached_time", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "'0.00000001'", + "got": "'0.00000001'", + "got_raw": "'0.00000001'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "currency_to_satoshi(1, 'jpy')", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.08333333333333333, + "edit_similarity": 0.0 + }, + { + "repo": "ofek/bit", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ofek/bit", + "expected": "1", + "got": "n._host._url", + "got_raw": "n._host._url", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.0 + }, + { + "repo": "ofek/bit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "100", + "got": "100", + "got_raw": "100", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "1024", + "got": "1024", + "got_raw": "1024", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "2048", + "got": "2048", + "got_raw": "2048", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"tool1\"", + "got": "\"tool1\"", + "got_raw": "\"tool1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"server2\"", + "got": "\"server2\"", + "got_raw": "\"server2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"prompt1\"", + "got": "\"prompt1\"", + "got_raw": "\"prompt1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "event.raw_request", + "got": "event.raw_request", + "got_raw": "event.raw_request", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"TOOL_INVOCATION\"", + "got": "AccessEventType.TOOL_INVOCATION", + "got_raw": "AccessEventType.TOOL_INVOCATION", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.625 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"npx\"", + "got": "\"npx\"", + "got_raw": "\"npx\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "result.output", + "got": "result.output", + "got_raw": "result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "1", + "got": "-1", + "got_raw": "-1", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"mcpm\"", + "got": "\"test-server\"", + "got_raw": "\"test-server\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.21052631578947367 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"Windsurf\")", + "got": "\"test\")", + "got_raw": "\"test\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"npx\")", + "got": "\"npx\")", + "got_raw": "\"npx\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"bunx\")", + "got": "\"bunx\")", + "got_raw": "\"bunx\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "result.output", + "got": "result.output", + "got_raw": "result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "result.output", + "got": "result.output", + "got_raw": "result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"node\"", + "got": "\"node\"", + "got_raw": "\"node\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"python\"", + "got": "'\"python", + "got_raw": "'\"python -m test_server\"'", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.875 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "[\"server.js\"]", + "got": "[\"server.js\"]", + "got_raw": "[\"server.js\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "middleware_types", + "got": "middleware_types", + "got_raw": "middleware_types", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "config.mcpServers", + "got": "config.mcpServers", + "got_raw": "config.mcpServers", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "[\"-m\", \"test_server\"]", + "got": "[\"-m\", \"test_server\"]", + "got_raw": "[\"-m\", \"test_server\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"https://api.example.com/mcp\"", + "got": "\"https://api.example.com/mcp\"", + "got_raw": "\"https://api.example.com/mcp\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "{\"Authorization\": \"Bearer token123\"}", + "got": "{\"Authorization\": \"Bearer token123\"}", + "got_raw": "{\"Authorization\": \"Bearer token123\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "ValueError, match=\"No supported servers to proxy\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.36065573770491804 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "{\"MCPM_STDIO_SERVER\": \"true\", \"API_KEY\": \"test123\"}", + "got": "{\"API_KEY\": \"test123\"}", + "got_raw": "{\"API_KEY\": \"test123\"}", + "exact_match": false, + "code_bleu": 0.44181707153725036, + "edit_similarity": 0.6027397260273972 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "{}", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "result.output", + "got": "result.output", + "got_raw": "result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "result.output.lower()", + "got": "result.output", + "got_raw": "result.output", + "exact_match": false, + "code_bleu": 0.36038417796587763, + "edit_similarity": 0.7647058823529411 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "result.output", + "got": "result.output", + "got_raw": "result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"nonexistent\")", + "got": "\"nonexistent\")", + "got_raw": "\"nonexistent\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"test-server\")", + "got": "\"test-server\")", + "got_raw": "\"test-server\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "stashed", + "got": "stashed", + "got_raw": "stashed", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "profiles", + "got": "profiles", + "got_raw": "profiles", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"%web-dev\"", + "got": "\"web-dev\"", + "got_raw": "\"web-dev\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "backup_names", + "got": "backup_names", + "got_raw": "backup_names", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "readme_content", + "got": "readme_content", + "got_raw": "readme_content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "stashed[\"claude-desktop\"]", + "got": "stashed[\"claude-desktop\"]", + "got_raw": "stashed[\"claude-desktop\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "analysis[\"clients_with_stashed\"]", + "got": "analysis[\"active_client\"]", + "got_raw": "analysis[\"active_client\"]", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.631578947368421 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "[cmd.name for cmd in main.commands.values()]", + "got": "main", + "got_raw": "main", + "exact_match": false, + "code_bleu": 0.03964023716675737, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "result.output", + "got": "result.output", + "got_raw": "result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "0", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "{}", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "[0, 1]", + "got": "[0, 1]", + "got_raw": "[0, 1]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"server2\")", + "got": "server2)", + "got_raw": "server2)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "result.output", + "got": "result.output", + "got_raw": "result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "result.output", + "got": "result.output", + "got_raw": "result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "5", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "130", + "got": "-1", + "got_raw": "-1", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.4 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"run\")", + "got": "\"test-server\")", + "got_raw": "\"test-server\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"ERROR\"", + "got": "\"INFO\"", + "got_raw": "\"INFO\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"WARNING\"", + "got": "\"INFO\"", + "got_raw": "\"INFO\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "caplog.text", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "None)", + "got": "None)", + "got_raw": "None)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"test\")", + "got": "\"test\")", + "got_raw": "\"test\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"github\")", + "got": "\"github\")", + "got_raw": "\"github\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "result.output", + "got": "result.output", + "got_raw": "result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "9999", + "got": "9999", + "got_raw": "9999", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"global\"", + "got": "\"test-server\"", + "got_raw": "\"test-server\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "mock_config", + "got": "mock_config", + "got_raw": "mock_config", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "result.output", + "got": "result.output", + "got_raw": "result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "profiles", + "got": "profiles", + "got_raw": "profiles", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"server-1\"", + "got": "\"server-1\"", + "got_raw": "\"server-1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"test-server\"", + "got": "\"test-server\"", + "got_raw": "\"test-server\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"sk-test-123\"", + "got": "\"sk-test-123\"", + "got_raw": "\"sk-test-123\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"shared-server\"", + "got": "\"shared-server\"", + "got_raw": "\"shared-server\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"sk-complete-123\"", + "got": "\"sk-complete-123\"", + "got_raw": "\"sk-complete-123\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "servers", + "got": "servers", + "got_raw": "servers", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"test-server\"", + "got": "\"test-server\"", + "got_raw": "\"test-server\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"stdio-server\"", + "got": "\"stdio-server\"", + "got_raw": "\"stdio-server\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"computercontroller\"", + "got": "\"computercontroller\"", + "got_raw": "\"computercontroller\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "{}", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "config", + "got": "config", + "got_raw": "config", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"qwen\")", + "got": "\"qwen\")", + "got_raw": "\"qwen\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"openai\"", + "got": "\"Qwen\"", + "got_raw": "\"Qwen\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"qwen-cli\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"Qwen CLI\"", + "got": "\"Qwen\"", + "got_raw": "\"Qwen\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"npx\"", + "got": "\"npx\"", + "got_raw": "\"npx\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "config", + "got": "config", + "got_raw": "config", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"npx\"", + "got": "\"npx\"", + "got_raw": "\"npx\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "servers", + "got": "servers", + "got_raw": "servers", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "server_list", + "got": "server_list", + "got_raw": "server_list", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"sample-key\"", + "got": "sample_server_config.env[\"API_KEY\"]", + "got_raw": "sample_server_config.env[\"API_KEY\"]", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.3404255319148936 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"test-value\"", + "got": "\"test-value\"", + "got_raw": "\"test-value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pathintegral-institute/mcpm.sh", + "expected": "\"test-server\"", + "got": "\"test-server\"", + "got_raw": "\"test-server\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "fn(x) - c)", + "got": "fn(x) - c)", + "got_raw": "fn(x) - c)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "fn(torch.full_like(x, c)))", + "got": "fn(torch.full_like(x, c)))", + "got_raw": "fn(torch.full_like(x, c)))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "alpha * y1 + (1 - alpha) * y2", + "got": "alpha * y1 + (1 - alpha) * y2", + "got_raw": "alpha * y1 + (1 - alpha) * y2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "parser[\"tool.poetry\"][\"version\"].replace('\"', \"\")", + "got": "parser[\"tool.poetry\"][\"version\"]", + "got_raw": "parser[\"tool.poetry\"][\"version\"]", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7901234567901234 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "targets)", + "got": "targets)", + "got_raw": "targets)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "RuntimeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "spot)", + "got": "moneyness)", + "got_raw": "moneyness)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "ValueError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "volatility)", + "got": "variance)", + "got_raw": "variance)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "time_to_maturity)", + "got": "time_to_maturity)", + "got_raw": "time_to_maturity)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "dtype", + "got": "dtype", + "got_raw": "dtype", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "f[0])", + "got": "spot[...", + "got_raw": "spot[..., 0])", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "\"spot\"", + "got": "\"Spot()\"", + "got_raw": "\"Spot()\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "\"zeros\"", + "got": "\"Zeros()\"", + "got_raw": "\"Zeros()\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "\"empty\"", + "got": "\"Empty()\"", + "got_raw": "\"Empty()\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "device)", + "got": "device)", + "got_raw": "device)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "KeyError)", + "got": "KeyError)", + "got_raw": "KeyError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "feature.__class__", + "got": "feature", + "got_raw": "feature", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5833333333333334 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "dtype", + "got": "dtype", + "got_raw": "dtype", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "DeprecationWarning)", + "got": "DeprecationWarning)", + "got_raw": "DeprecationWarning)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.device(device)", + "got": "device", + "got_raw": "device", + "exact_match": false, + "code_bleu": 0.09910059291689341, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "european.payoff())", + "got": "european.payoff())", + "got_raw": "european.payoff())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.tensor([2.0, 1.0, 0.0]).to(device))", + "got": "spot)", + "got_raw": "spot)", + "exact_match": false, + "code_bleu": 0.05969832954575233, + "edit_similarity": 0.17391304347826086 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.tensor([1.0, 2.0, 5.0]).to(device))", + "got": "torch.tensor([1.0, 0.0, 0.0]).to(device))", + "got_raw": "torch.tensor([1.0, 0.0, 0.0]).to(device))", + "exact_match": false, + "code_bleu": 0.744373319312051, + "edit_similarity": 0.9512195121951219 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "dtype", + "got": "dtype", + "got_raw": "dtype", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "spot)", + "got": "spot)", + "got_raw": "spot)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "dtype", + "got": "dtype", + "got_raw": "dtype", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "DeprecationWarning)", + "got": "DeprecationWarning)", + "got_raw": "DeprecationWarning)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.device(device)", + "got": "torch.device(device)", + "got_raw": "torch.device(device)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "dtype", + "got": "dtype", + "got_raw": "dtype", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "DeprecationWarning)", + "got": "DeprecationWarning)", + "got_raw": "DeprecationWarning)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "DeprecationWarning)", + "got": "DeprecationWarning)", + "got_raw": "DeprecationWarning)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "DeprecationWarning)", + "got": "DeprecationWarning)", + "got_raw": "DeprecationWarning)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "dtype", + "got": "dtype", + "got_raw": "dtype", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "KeyError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "TypeError)", + "got": "AttributeError)", + "got_raw": "AttributeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.56 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((10, 3))", + "got": "(10, 2)", + "got_raw": "(10, 2)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((10, 4))", + "got": "torch.Size((10, 4))", + "got_raw": "torch.Size((10, 4))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((10, 3))", + "got": "(10, 2)", + "got_raw": "(10, 2)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((10, 4))", + "got": "torch.Size((10, 4))", + "got_raw": "torch.Size((10, 4))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.ones_like(stock.spot))", + "got": "torch.zeros_like(stock.spot))", + "got_raw": "torch.zeros_like(stock.spot))", + "exact_match": false, + "code_bleu": 0.7071067811865475, + "edit_similarity": 0.9122807017543859 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.zeros_like(stock.volatility))", + "got": "torch.zeros_like(stock.volatility))", + "got_raw": "torch.zeros_like(stock.volatility))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((10, 3))", + "got": "(10, 2)", + "got_raw": "(10, 2)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((10, 4))", + "got": "torch.Size((10, 4))", + "got_raw": "torch.Size((10, 4))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((10, 3))", + "got": "(10, 2)", + "got_raw": "(10, 2)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((10, 4))", + "got": "torch.Size((10, 4))", + "got_raw": "torch.Size((10, 4))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((10, 3))", + "got": "(10, 2)", + "got_raw": "(10, 2)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((10, 4))", + "got": "torch.Size((10, 4))", + "got_raw": "torch.Size((10, 4))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "i1)", + "got": "i1)", + "got_raw": "i1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "i2)", + "got": "i2)", + "got_raw": "i2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "i3)", + "got": "i3)", + "got_raw": "i3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "i4)", + "got": "i4)", + "got_raw": "i4)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "-payoff)", + "got": "payoff)", + "got_raw": "payoff)", + "exact_match": false, + "code_bleu": 0.4714045207910317, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N,))", + "got": "(N,", + "got_raw": "(N, 1)", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.3157894736842105 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, H, T))", + "got": "(N, T, H)", + "got_raw": "(N, T, H)", + "exact_match": false, + "code_bleu": 0.2724971536747834, + "edit_similarity": 0.4666666666666667 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, 1, h_in))", + "got": "(T + 1, N, 1)", + "got_raw": "(T + 1, N, 1)", + "exact_match": false, + "code_bleu": 0.24350093660632421, + "edit_similarity": 0.32432432432432434 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "a)", + "got": "a)", + "got_raw": "a)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "c)", + "got": "c)", + "got_raw": "c)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "x2)", + "got": "x2)", + "got_raw": "x2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "eta)", + "got": "eta)", + "got_raw": "eta)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "loss)", + "got": "loss)", + "got_raw": "loss)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "in_expect", + "got": "in_expect", + "got_raw": "in_expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "out_expect", + "got": "out_expect", + "got_raw": "out_expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "out_features", + "got": "out_features", + "got_raw": "out_features", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "2 * (n_layers + 1)", + "got": "n_layers", + "got_raw": "n_layers", + "exact_match": false, + "code_bleu": 0.08494336535733721, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, H_out))", + "got": "(N, H_out)", + "got_raw": "(N, H_out)", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.625 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, M_1, H_out))", + "got": "torch.Size((N, M_1, H_out))", + "got_raw": "torch.Size((N, M_1, H_out))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, M_1, M_2, H_out))", + "got": "torch.Size((N, M_1, M_2, H_out))", + "got_raw": "torch.Size((N, M_1, M_2, H_out))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, H_out))", + "got": "(N, H_out)", + "got_raw": "(N, H_out)", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.625 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, M_1, H_out))", + "got": "torch.Size((N, M_1, H_out))", + "got_raw": "torch.Size((N, M_1, H_out))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, M_1, M_2, H_out))", + "got": "torch.Size((N, M_1, M_2, H_out))", + "got_raw": "torch.Size((N, M_1, M_2, H_out))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "\"SVIVariance(a=1.0, b=0, rho=0.1, m=0.2, sigma=0.3)\"", + "got": "\"SVIVariance(a=1.0, b=0.0, rho=0.1, m=0.2, sigma=0.3)\"", + "got_raw": "\"SVIVariance(a=1.0, b=0.0, rho=0.1, m=0.2, sigma=0.3)\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9811320754716981 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, 1))", + "got": "(N, M_1)", + "got_raw": "(N, M_1)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, M_1, 1))", + "got": "torch.Size((N, M_1, 1))", + "got_raw": "torch.Size((N, M_1, 1))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, M_1, M_2, 1))", + "got": "torch.Size((N, M_1, M_2, 1))", + "got_raw": "torch.Size((N, M_1, M_2, 1))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N,))", + "got": "(N,", + "got_raw": "(N, M_1)", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.3157894736842105 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, 1))", + "got": "torch.Size((N, M_1, M_2))", + "got_raw": "torch.Size((N, M_1, M_2))", + "exact_match": false, + "code_bleu": 0.6115380576901023, + "edit_similarity": 0.8372093023255814 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, M_1))", + "got": "torch.Size((N, M_1))", + "got_raw": "torch.Size((N, M_1))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, M_1, 1))", + "got": "torch.Size((N, M_1))", + "got_raw": "torch.Size((N, M_1))", + "exact_match": false, + "code_bleu": 0.7093467443268724, + "edit_similarity": 0.9302325581395349 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, M_1, M_2))", + "got": "torch.Size((N, M_1, M_2))", + "got_raw": "torch.Size((N, M_1, M_2))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, M_1, M_2, 1))", + "got": "torch.Size((N, M_1, M_2, 1))", + "got_raw": "torch.Size((N, M_1, M_2, 1))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "m)", + "got": "m)", + "got_raw": "m)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "1.0", + "got": "derivative.strike", + "got_raw": "derivative.strike", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.1 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "2.0", + "got": "2.0", + "got_raw": "2.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "BSEuropeanOption", + "got": "BlackScholes", + "got_raw": "BlackScholes", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.21428571428571427 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, 1))", + "got": "(N, M_1)", + "got_raw": "(N, M_1)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, M_1, 1))", + "got": "torch.Size((N, M_1, 1))", + "got_raw": "torch.Size((N, M_1, 1))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((N, M_1, M_2, 1))", + "got": "torch.Size((N, M_1, M_2, 1))", + "got_raw": "torch.Size((N, M_1, M_2, 1))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "m)", + "got": "m)", + "got_raw": "m)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "m)", + "got": "m)", + "got_raw": "m)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "price)", + "got": "price)", + "got_raw": "price)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "m)", + "got": "m)", + "got_raw": "m)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.float32", + "got": "torch.float32", + "got_raw": "torch.float32", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.float64", + "got": "torch.float64", + "got_raw": "torch.float64", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((n_paths, n_steps))", + "got": "(n_paths, n_steps + 1, 2)", + "got_raw": "(n_paths, n_steps + 1, 2)", + "exact_match": false, + "code_bleu": 0.3700202552104677, + "edit_similarity": 0.6545454545454545 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.float32", + "got": "torch.float32", + "got_raw": "torch.float32", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.float64", + "got": "torch.float64", + "got_raw": "torch.float64", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "output.variance.sqrt())", + "got": "torch.randn_like(output.volatility))", + "got_raw": "torch.randn_like(output.volatility))", + "exact_match": false, + "code_bleu": 0.25406637407730737, + "edit_similarity": 0.4067796610169492 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "ValueError, match=\"jump_mean_down must be postive\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.3548387096774194 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "ValueError, match=\"jump prob must be in 0 and 1 incl\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.3384615384615385 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "ValueError, match=\"jump_mean_up must be postive and smaller than 1\")", + "got": "", + "got_raw": "", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect)", + "got": "expect)", + "got_raw": "expect)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.float32", + "got": "torch.float32", + "got_raw": "torch.float32", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.float64", + "got": "torch.float64", + "got_raw": "torch.float64", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.Size((n_paths, n_steps))", + "got": "(n_paths, n_steps)", + "got_raw": "(n_paths, n_steps)", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "ValueError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "torch.zeros_like(output[0]))", + "got": "torch.zeros_like(output.mean(0)))", + "got_raw": "torch.zeros_like(output.mean(0)))", + "exact_match": false, + "code_bleu": 0.4408231875586727, + "edit_similarity": 0.8524590163934426 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "iv_expected)", + "got": "iv_expected)", + "got_raw": "iv_expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet-research/pfhedge", + "expected": "output.variance.sqrt())", + "got": "torch.randn_like(output.volatility))", + "got_raw": "torch.randn_like(output.volatility))", + "exact_match": false, + "code_bleu": 0.25406637407730737, + "edit_similarity": 0.4067796610169492 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "obj", + "got": "obj", + "got_raw": "obj", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "str", + "got": "str", + "got_raw": "str", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "int", + "got": "int", + "got_raw": "int", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "env", + "got": "env", + "got_raw": "env", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "val2", + "got": "val2", + "got_raw": "val2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "impl", + "got": "impl", + "got_raw": "impl", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "output", + "got": "output", + "got_raw": "output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "content", + "got": "content", + "got_raw": "content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "content", + "got": "content", + "got_raw": "content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "ts_code", + "got": "ts_code", + "got_raw": "ts_code", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "ts_type", + "got": "ts_type", + "got_raw": "ts_type", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "ts_code", + "got": "ts_code", + "got_raw": "ts_code", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "ts_code", + "got": "ts_code", + "got_raw": "ts_code", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "ts_code", + "got": "ts_code", + "got_raw": "ts_code", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "ts_code", + "got": "ts_code", + "got_raw": "ts_code", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "min_time", + "got": "min_time", + "got_raw": "min_time", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "max_time", + "got": "max_time", + "got_raw": "max_time", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "view_base", + "got": "view_base", + "got_raw": "view_base", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "log_level", + "got": "log_level", + "got_raw": "log_level", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "root_path", + "got": "root_path", + "got_raw": "root_path", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "error_str", + "got": "error_str", + "got_raw": "error_str", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "child_idx", + "got": "child_idx", + "got_raw": "child_idx", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "responses", + "got": "responses", + "got_raw": "responses", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "log_output", + "got": "log_output", + "got_raw": "log_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "lock_second", + "got": "lock_second", + "got_raw": "lock_second", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "logger_name", + "got": "logger_name", + "got_raw": "logger_name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "ValueError)", + "got": "AttributeError)", + "got_raw": "AttributeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "ValueError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "mock_session", + "got": "mock_session", + "got_raw": "mock_session", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "test_content", + "got": "test_content", + "got_raw": "test_content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "expected_url", + "got": "expected_url", + "got_raw": "expected_url", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "ExampleModel", + "got": "list", + "got_raw": "list", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.125 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "expected_str", + "got": "expected_str", + "got_raw": "expected_str", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "expected_copy", + "got": "expected_copy", + "got_raw": "expected_copy", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "expected_tags", + "got": "expected_tags", + "got_raw": "expected_tags", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "TimeoutError)", + "got": "V8RuntimeError)", + "got_raw": "V8RuntimeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6428571428571429 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "response_data", + "got": "response_data", + "got_raw": "response_data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "error_payload", + "got": "error_payload", + "got_raw": "error_payload", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "response_data", + "got": "response_data", + "got_raw": "response_data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "main_model_idx", + "got": "main_model_idx", + "got_raw": "main_model_idx", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "expected_prefix", + "got": "expected_prefix", + "got_raw": "expected_prefix", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "child_model_idx", + "got": "child_model_idx", + "got_raw": "child_model_idx", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "expected_ignore", + "got": "expected_ignore", + "got_raw": "expected_ignore", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "ValidationError)", + "got": "ValidationError)", + "got_raw": "ValidationError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "expected_is_file", + "got": "expected_is_file", + "got_raw": "expected_is_file", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "args[0]", + "got": "args[0]", + "got_raw": "args[0]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "args[1]", + "got": "args[1]", + "got_raw": "args[1]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "args[2]", + "got": "kwargs[\"env\"][\"PATH\"]", + "got_raw": "kwargs[\"env\"][\"PATH\"]", + "exact_match": false, + "code_bleu": 0.20556680845025982, + "edit_similarity": 0.42857142857142855 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "result[0]", + "got": "result[0]", + "got_raw": "result[0]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "captured.out", + "got": "captured.err", + "got_raw": "captured.err", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.75 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "result.stdout", + "got": "result.stdout", + "got_raw": "result.stdout", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "tmp_path / \"views\"", + "got": "tmp_path", + "got_raw": "tmp_path", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "root_path / \"_ssr\"", + "got": "root_path / \"_ssr\"", + "got_raw": "root_path / \"_ssr\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "expected[key].alias", + "got": "expected[key].alias", + "got_raw": "expected[key].alias", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "interface.typehints", + "got": "interface.params", + "got_raw": "interface.params", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6857142857142857 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "CallbackType.CREATED", + "got": "CallbackType.added", + "got_raw": "CallbackType.added", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6842105263157895 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "CallbackType.DELETED", + "got": "CallbackType.DELETED", + "got_raw": "CallbackType.DELETED", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "root_path / \"_server\"", + "got": "ManagedViewPath.from_view_root(tmpdir)", + "got_raw": "ManagedViewPath.from_view_root(tmpdir)", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.2033898305084746 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "root_path / \"_static\"", + "got": "root_path / \"_static\"", + "got_raw": "root_path / \"_static\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "expected[key].default", + "got": "expected[key].default", + "got_raw": "expected[key].default", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "CallbackType.MODIFIED", + "got": "watcher._map_change_to_callback_type(Change.modified)", + "got_raw": "watcher._map_change_to_callback_type(Change.modified)", + "exact_match": false, + "code_bleu": 0.16036590969929357, + "edit_similarity": 0.2972972972972973 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "log_output[\"exception\"]", + "got": "log_output[\"exception\"]", + "got_raw": "log_output[\"exception\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "interface.response_type", + "got": "interface.typehints", + "got_raw": "interface.typehints", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "session_cache.loop_locks", + "got": "session_cache", + "got_raw": "session_cache", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.7027027027027027 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "expected[key].annotation", + "got": "expected[key].annotation", + "got_raw": "expected[key].annotation", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "result_model.model_fields", + "got": "result_model.__fields__", + "got_raw": "result_model.__fields__", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "root_path / \"detail/page.tsx\"", + "got": "\"/detail/page.tsx\"", + "got_raw": "\"/detail/page.tsx\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.723404255319149 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "metadata.static_artifact_shas", + "got": "metadata", + "got_raw": "metadata", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.43243243243243246 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "FunctionActionType.PASSTHROUGH", + "got": "FunctionActionType.PASSTHROUGH", + "got_raw": "FunctionActionType.PASSTHROUGH", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "interface.include_superclasses", + "got": "interface", + "got_raw": "interface", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "ssr_base / \"stub_controller.js\"", + "got": "ssr_base", + "got_raw": "ssr_base", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.41025641025641024 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "package_dir_path / package_name", + "got": "site_packages_path", + "got_raw": "site_packages_path", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.4897959183673469 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "TypeError, match=\"1 is not a string\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.17909498863725698, + "edit_similarity": 0.2916666666666667 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "[\"age\"]", + "got": "\"body\"", + "got_raw": "\"body\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "{\"a\": 4}", + "got": "{\"a\": calculate_primes(10)}", + "got_raw": "{\"a\": calculate_primes(10)}", + "exact_match": false, + "code_bleu": 0.3155984539112945, + "edit_similarity": 0.4 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "{\"a\": 0}", + "got": "{\"a\": 0}", + "got_raw": "{\"a\": 0}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "{\"a\": 20}", + "got": "ExampleResponse(a=10.5)", + "got_raw": "ExampleResponse(a=10.5)", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.125 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "{\"fastapi\"}", + "got": "set([\"fasta\"])", + "got_raw": "set([\"fasta\"])", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.56 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "{\"a\": 1244}", + "got": "{\"a\": 1244}", + "got_raw": "{\"a\": 1244}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "(\"test_webapp\",)", + "got": "(MockConfig(),)", + "got_raw": "(MockConfig(),)", + "exact_match": false, + "code_bleu": 0.31239399369202553, + "edit_similarity": 0.1935483870967742 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "{\"resolved_dep\"}", + "got": "{", + "got_raw": "{ \"payload\", \"request\", }", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "{\"job1\", \"job2\"}", + "got": "{\"job1\", \"job2\"}", + "got_raw": "{\"job1\", \"job2\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "(\"line1\", \"line2\")", + "got": "[\"line1\", \"line2\"]", + "got_raw": "[\"line1\", \"line2\"]", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "{\"resolved_dep\": 1}", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.18072040072196896, + "edit_similarity": 0.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "{\"a\": 1244, \"b\": 78503}", + "got": "optimized_func()", + "got_raw": "optimized_func()", + "exact_match": false, + "code_bleu": 0.12663928094193208, + "edit_similarity": 0.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "{\"raw_value\": \"success\"}", + "got": "{\"raw_value\": \"success\"}", + "got_raw": "{\"raw_value\": \"success\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "(None, ResponseModelType.SINGLE_RESPONSE)", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.08494336535733721, + "edit_similarity": 0.17777777777777778 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "(ExampleModel, ResponseModelType.SINGLE_RESPONSE)", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.07142857142857142, + "edit_similarity": 0.11320754716981132 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "{ T: int, K: str, }", + "got": "{ \"T\": TypeVar(\"T\") }", + "got_raw": "{ \"T\": TypeVar(\"T\") }", + "exact_match": false, + "code_bleu": 0.15256662465462703, + "edit_similarity": 0.4 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "{ K: int, T: int, }", + "got": "{K, T})", + "got_raw": "{K, T})", + "exact_match": false, + "code_bleu": 0.22956176497670683, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "{ \"pydantic\", \"pydantic_core\", \"pydantic-settings\", }", + "got": "set()", + "got_raw": "set()", + "exact_match": false, + "code_bleu": 0.1424691910596736, + "edit_similarity": 0.10344827586206896 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "{ str(Path(path).absolute()) for path in expected_paths }", + "got": "set(expected_paths)", + "got_raw": "set(expected_paths)", + "exact_match": false, + "code_bleu": 0.10053811915205507, + "edit_similarity": 0.4473684210526316 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "[ CallbackType.CREATED, CallbackType.MODIFIED, CallbackType.DELETED, ]", + "got": "[CallbackType.CREATED, CallbackType.MODIFIED, CallbackType.DELETED]", + "got_raw": "[CallbackType.CREATED, CallbackType.MODIFIED, CallbackType.DELETED]", + "exact_match": false, + "code_bleu": 0.8696199183331033, + "edit_similarity": 0.9781021897810219 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "{ \"project\": { \"name\": \"mountaineer\", \"version\": \"0.2.0\", \"description\": \"\", }, }", + "got": "{ \"project\": { \"name\": \"mountaineer\", \"version\": \"0.2.0\", \"description\": \"\", } }", + "got_raw": "{ \"project\": { \"name\": \"mountaineer\", \"version\": \"0.2.0\", \"description\": \"\", } }", + "exact_match": false, + "code_bleu": 0.9051563294567685, + "edit_similarity": 0.9937888198757764 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "{ \"tool\": { \"poetry\": { \"name\": \"mountaineer\", \"version\": \"0.2.0\", \"description\": \"\", }, }, }", + "got": "{ \"tool\": { \"poetry\": { \"version\": \"0.2.0\", } } }", + "got_raw": "{ \"tool\": { \"poetry\": { \"version\": \"0.2.0\", } } }", + "exact_match": false, + "code_bleu": 0.4090365494300435, + "edit_similarity": 0.676056338028169 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "[ 'data: {\"passthrough\": {\"value\": \"Hello\"}}', 'data: {\"passthrough\": {\"value\": \"World\"}}', ]", + "got": "[ b\"1\", b\"2\", b\"3\", b\"4\", b\"5\", b\"6\", b\"7\", b\"8\", b\"9\", b\"10\", b\"11\", b\"12\", b\"13\", b\"14\", b\"15\", b\"16\", b\"17\", b\"18\", b\"19\", b\"20\", b\"21\", b\"22\", b\"23\", b\"24\", b\"25\", b\"26\", b\"27\", b", + "got_raw": "[ b\"1\", b\"2\", b\"3\", b\"4\", b\"5\", b\"6\", b\"7\", b\"8\", b\"9\", b\"10\", b\"11\", b\"12\", b\"13\", b\"14\", b\"15\", b\"16\", b\"17\", b\"18\", b\"19\", b\"20\", b\"21\", b\"22\", b\"23\", b\"24\", b\"25\", b\"26\", b\"27\", b", + "exact_match": false, + "code_bleu": 0.017143556986705912, + "edit_similarity": 0.07246376811594203 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "1", + "got": "1.0", + "got_raw": "1.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "0", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "3", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "4", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "1", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "6", + "got": "6", + "got_raw": "6", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"/\"", + "got": "\"/\"", + "got_raw": "\"/\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"7\"", + "got": "\"seven\"", + "got_raw": "\"seven\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"8\"", + "got": "\"eight\"", + "got_raw": "\"eight\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"9\"", + "got": "9", + "got_raw": "9", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"abc\"", + "got": "\"abc\"", + "got_raw": "\"abc\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"def\"", + "got": "\"def\"", + "got_raw": "\"def\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"any\"", + "got": "\"str\"", + "got_raw": "\"str\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"null\"", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"0.1.0\"", + "got": "\"0.0.0\"", + "got_raw": "\"0.0.0\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"custom\"", + "got": "\"custom\"", + "got_raw": "\"custom\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"layout\"", + "got": "\"layout\"", + "got_raw": "\"layout\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"Status\"", + "got": "\"test_module.Status\"", + "got_raw": "\"test_module.Status\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"Test 402\"", + "got": "\"Test 402\"", + "got_raw": "\"Test 402\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"standard\"", + "got": "\"standard\"", + "got_raw": "\"standard\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"TestModel\"", + "got": "\"test.module.TestModel\"", + "got_raw": "\"test.module.TestModel\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6470588235294118 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"UserModel\"", + "got": "\"UserModel\"", + "got_raw": "\"UserModel\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"response1\"", + "got": "\"response1\"", + "got_raw": "\"response1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"response2\"", + "got": "\"response2\"", + "got_raw": "\"response2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"test-value\"", + "got": "\"test-value\"", + "got_raw": "\"test-value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"StatusEnum\"", + "got": "\"StatusEnum\"", + "got_raw": "\"StatusEnum\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"int_parsing\"", + "got": "\"validation\"", + "got_raw": "\"validation\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"/test-page/\"", + "got": "\"/test-layout.tsx\"", + "got_raw": "\"/test-layout.tsx\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5806451612903226 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"/some-page/\"", + "got": "\"/some-page/\"", + "got_raw": "\"/some-page/\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"Set\"", + "got": "\"Set[float]\"", + "got_raw": "\"Set[float]\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"SimpleModel\"", + "got": "\"SimpleModel\"", + "got_raw": "\"SimpleModel\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"not_a_number\"", + "got": "\"not_a_number\"", + "got_raw": "\"not_a_number\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"controller_a\"", + "got": "\"renders.a\"", + "got_raw": "\"renders.a\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.48 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"controller_b\"", + "got": "\"controller_b\"", + "got_raw": "\"controller_b\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"layout-value\"", + "got": "\"layout-value\"", + "got_raw": "\"layout-value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"ContainerStr\"", + "got": "\"StringContainer\"", + "got_raw": "\"StringContainer\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"ContainerInt\"", + "got": "\"ContainerInt\"", + "got_raw": "\"ContainerInt\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"don't do this\"", + "got": "\"7\"", + "got_raw": "\"7\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"simple_action\"", + "got": "\"simple_action\"", + "got_raw": "\"simple_action\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"Array\"", + "got": "\"list[str]\"", + "got_raw": "\"list[str]\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.38461538461538464 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"CircularModel\"", + "got": "\"SimpleModel\"", + "got_raw": "\"SimpleModel\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6428571428571429 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"processed CSS\"", + "got": "\"processed CSS\"", + "got_raw": "\"processed CSS\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"late response\"", + "got": "\"late response\"", + "got_raw": "\"late response\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"AuthModels_User\"", + "got": "\"AuthModels_User\"", + "got_raw": "\"AuthModels_User\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"TreeModels_Node\"", + "got": "\"Node\"", + "got_raw": "\"Node\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5217391304347826 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"ExampleException\"", + "got": "\"ExampleException\"", + "got_raw": "\"ExampleException\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"UsersModels_User\"", + "got": "\"User\"", + "got_raw": "\"User\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"process-response\"", + "got": "\"process-response\"", + "got_raw": "\"process-response\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"get_external_data\"", + "got": "\"get_external_data\"", + "got_raw": "\"get_external_data\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"text/event-stream\"", + "got": "\"application/json\"", + "got_raw": "\"application/json\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.16216216216216217 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "\"session_test_value\"", + "got": "\"session_test_value\"", + "got_raw": "\"session_test_value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "id(new_path)", + "got": "id(new_path)", + "got_raw": "id(new_path)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "set(expected)", + "got": "set(expected)", + "got_raw": "set(expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "expected.keys()", + "got": "expected.keys()", + "got_raw": "expected.keys()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "interface.to_js()", + "got": "interface.include_superclasses", + "got_raw": "interface.include_superclasses", + "exact_match": false, + "code_bleu": 0.3567621345008163, + "edit_similarity": 0.5106382978723404 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "interface.to_js()", + "got": "interface", + "got_raw": "interface", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.6923076923076923 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "str(HelloEnum.eight)", + "got": "\"eight\"", + "got_raw": "\"eight\"", + "exact_match": false, + "code_bleu": 0.08333333333333333, + "edit_similarity": 0.37037037037037035 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "error.message.lower()", + "got": "error.error", + "got_raw": "error.error", + "exact_match": false, + "code_bleu": 0.2548300960720116, + "edit_similarity": 0.5 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "str(GoodbyeEnum.nine)", + "got": "9", + "got_raw": "9", + "exact_match": false, + "code_bleu": 0.08333333333333333, + "edit_similarity": 0.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "str(new_path.root_link)", + "got": "str(new_path.root_link)", + "got_raw": "str(new_path.root_link)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "expected[key].is_required()", + "got": "expected[key].is_required()", + "got_raw": "expected[key].is_required()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "str(new_path.package_root_link)", + "got": "str(new_path.package_root_link)", + "got_raw": "str(new_path.package_root_link)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "ExamplePassthroughModel(status=\"success\")", + "got": "passthrough", + "got_raw": "passthrough", + "exact_match": false, + "code_bleu": 0.08333333333333333, + "edit_similarity": 0.4230769230769231 + }, + { + "repo": "piercefreeman/mountaineer", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "1", + "got": "'1'", + "got_raw": "'1'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "pyamg/pyamg", + "expected": "7", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "3", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0", + "got": "0.0", + "got_raw": "0.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "pyamg/pyamg", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0", + "got": "{'niter':", + "got_raw": "{'niter': 2, 'info': 0}", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "2", + "got": "Anew", + "got_raw": "Anew", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "3", + "got": "b.shape[1]", + "got_raw": "b.shape[1]", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "4)", + "got": "0)", + "got_raw": "0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "pyamg/pyamg", + "expected": "1)", + "got": "1.0)", + "got_raw": "1.0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pyamg/pyamg", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "7)", + "got": "0)", + "got_raw": "0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "pyamg/pyamg", + "expected": "6)", + "got": "0)", + "got_raw": "0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "pyamg/pyamg", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "4)", + "got": "4)", + "got_raw": "4)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "7)", + "got": "5)", + "got_raw": "5)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "pyamg/pyamg", + "expected": "5)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "pyamg/pyamg", + "expected": "5)", + "got": "5)", + "got_raw": "5)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.7", + "got": "0.9", + "got_raw": "0.9", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.9", + "got": "1.0", + "got_raw": "1.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.4", + "got": "0.4", + "got_raw": "0.4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.7", + "got": "0.7", + "got_raw": "0.7", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.8", + "got": "0.8", + "got_raw": "0.8", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.9", + "got": "1.0", + "got_raw": "1.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.0", + "got": "0.0", + "got_raw": "0.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.2", + "got": "resid2", + "got_raw": "resid2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.3", + "got": "resid2", + "got_raw": "resid2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "11)", + "got": "12.5)", + "got_raw": "12.5)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.01", + "got": "-", + "got_raw": " - sa_new.levels[-1].A.toarray()))", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.25", + "got": "0.25", + "got_raw": "0.25", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.65", + "got": "0.01", + "got_raw": "0.01", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.45", + "got": "0.3", + "got_raw": "0.3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.85", + "got": "0.01", + "got_raw": "0.01", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.01", + "got": "1e-12", + "got_raw": "1e-12", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.45", + "got": "0.45", + "got_raw": "0.45", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.92", + "got": "0.92", + "got_raw": "0.92", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.01", + "got": "1e-12", + "got_raw": "1e-12", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.01", + "got": "1e-12", + "got_raw": "1e-12", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.20", + "got": "2.0)", + "got_raw": "2.0)", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.5 + }, + { + "repo": "pyamg/pyamg", + "expected": "1e-4", + "got": "1.0e-12", + "got_raw": "1.0e-12", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.0)", + "got": "0.0)", + "got_raw": "0.0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "1e-2", + "got": "tol", + "got_raw": "tol", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "1e-6", + "got": "1e-2", + "got_raw": "1e-2", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.75 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.0)", + "got": "0.0)", + "got_raw": "0.0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.0)", + "got": "case['tol'])", + "got_raw": "case['tol'])", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.125 + }, + { + "repo": "pyamg/pyamg", + "expected": "1e-8", + "got": "0.01", + "got_raw": "0.01", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.25 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.01", + "got": "resid2", + "got_raw": "resid2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.03", + "got": "resid2", + "got_raw": "resid2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.95", + "got": "1.0", + "got_raw": "1.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "pyamg/pyamg", + "expected": "1e-4", + "got": "1e-4", + "got_raw": "1e-4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "1e-7", + "got": "1e-8", + "got_raw": "1e-8", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.75 + }, + { + "repo": "pyamg/pyamg", + "expected": "1.0)", + "got": "-1.1", + "got_raw": "-1.1 / 9.5)", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.5 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.0)", + "got": "-1.0)", + "got_raw": "-1.0)", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.0)", + "got": "0.0)", + "got_raw": "0.0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.0)", + "got": "0.0)", + "got_raw": "0.0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "1e-12", + "got": "res[0])", + "got_raw": "res[0])", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "pyamg/pyamg", + "expected": "0.54)", + "got": "0.545351286587159)", + "got_raw": "0.545351286587159)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.43478260869565216 + }, + { + "repo": "pyamg/pyamg", + "expected": "n", + "got": "n", + "got_raw": "n", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "B)", + "got": "B)", + "got_raw": "B)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "L)", + "got": "L)", + "got_raw": "L)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "M)", + "got": "M)", + "got_raw": "M)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "K)", + "got": "K)", + "got_raw": "K)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "B)", + "got": "B)", + "got_raw": "B)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "c)", + "got": "c)", + "got_raw": "c)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "b)", + "got": "b)", + "got_raw": "b)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "c)", + "got": "c)", + "got_raw": "c)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "V2)", + "got": "V2)", + "got_raw": "V2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "E2)", + "got": "E2)", + "got_raw": "E2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "AA)", + "got": "AA)", + "got_raw": "AA)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "bb)", + "got": "bb)", + "got_raw": "bb)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "tol", + "got": "tol", + "got_raw": "tol", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "x1)", + "got": "x1)", + "got_raw": "x1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "x2)", + "got": "x2)", + "got_raw": "x2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "x3)", + "got": "x3)", + "got_raw": "x3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "A0)", + "got": "A0)", + "got_raw": "A0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "pc)", + "got": "[1", + "got_raw": "[1, 2, 3, 4, 5, 6, 7, 8, 9])", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "Bf)", + "got": "A@B)", + "got_raw": "A@B)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "pyamg/pyamg", + "expected": "x64)", + "got": "x64)", + "got_raw": "x64)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "rhs)", + "got": "rhs)", + "got_raw": "rhs)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "x38)", + "got": "x38)", + "got_raw": "x38)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "mis)", + "got": "mis)", + "got_raw": "mis)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "D_A)", + "got": "D_A)", + "got_raw": "D_A)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "soln)", + "got": "soln)", + "got_raw": "soln)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "D_AA)", + "got": "D_AA)", + "got_raw": "D_AA)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "rratio", + "got": "rratio", + "got_raw": "rratio", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "exact)", + "got": "exact)", + "got_raw": "exact)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "flag2)", + "got": "flag2)", + "got_raw": "flag2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "solver", + "got": "solver", + "got_raw": "solver", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "d_ref)", + "got": "d_ref)", + "got_raw": "d_ref)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "m_ref)", + "got": "m_ref)", + "got_raw": "m_ref)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "D_ref)", + "got": "D_ref)", + "got_raw": "D_ref)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "P_ref)", + "got": "P_ref)", + "got_raw": "P_ref)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "exact)", + "got": "exact)", + "got_raw": "exact)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "c_factor", + "got": "c_factor", + "got_raw": "c_factor", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "expected", + "got": "expected)", + "got_raw": "expected)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.9411764705882353 + }, + { + "repo": "pyamg/pyamg", + "expected": "c_factor", + "got": "c_factor", + "got_raw": "c_factor", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "P_exact)", + "got": "P_exact)", + "got_raw": "P_exact)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "R_exact)", + "got": "R_exact)", + "got_raw": "R_exact)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "stencil)", + "got": "isotropic=True)", + "got_raw": " isotropic=True), stencil)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.17391304347826086 + }, + { + "repo": "pyamg/pyamg", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "scipyres)", + "got": "scipyres[1:])", + "got_raw": "scipyres[1:])", + "exact_match": false, + "code_bleu": 0.24446151121745047, + "edit_similarity": 0.8181818181818182 + }, + { + "repo": "pyamg/pyamg", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "pinv_test)", + "got": "pinv_test)", + "got_raw": "pinv_test)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "A_expected)", + "got": "A_expected)", + "got_raw": "A_expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "B_expected)", + "got": "B_expected)", + "got_raw": "B_expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "d_expected)", + "got": "d_expected)", + "got_raw": "d_expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "m_expected)", + "got": "m_expected)", + "got_raw": "m_expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "expected_speed", + "got": "expected_speed)", + "got_raw": "expected_speed)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.9655172413793104 + }, + { + "repo": "pyamg/pyamg", + "expected": "fine_candidates)", + "got": "np.eye(coarse_candidates.shape[1]))", + "got_raw": "np.eye(coarse_candidates.shape[1]))", + "exact_match": false, + "code_bleu": 0.10390302174233558, + "edit_similarity": 0.5490196078431373 + }, + { + "repo": "pyamg/pyamg", + "expected": "splitting_paper)", + "got": "splitting_paper)", + "got_raw": "splitting_paper)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "0*B)", + "got": "np.zeros_like(B))", + "got_raw": "np.zeros_like(B))", + "exact_match": false, + "code_bleu": 0.24446151121745047, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "pyamg/pyamg", + "expected": "L + M)", + "got": "np.zeros_like(L))", + "got_raw": "np.zeros_like(L))", + "exact_match": false, + "code_bleu": 0.20556680845025982, + "edit_similarity": 0.17391304347826086 + }, + { + "repo": "pyamg/pyamg", + "expected": "out[1])", + "got": "out[1])", + "got_raw": "out[1])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "I2.data)", + "got": "I2.data)", + "got_raw": "I2.data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "fvals[i]", + "got": "fvals[i]", + "got_raw": "fvals[i]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "I2.indptr)", + "got": "I2.indptr)", + "got_raw": "I2.indptr)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "exact.data)", + "got": "exact.data)", + "got_raw": "exact.data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "100.0/100.0)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "pyamg/pyamg", + "expected": "225.0/100.0)", + "got": "100.0/100.0)", + "got_raw": "100.0/100.0)", + "exact_match": false, + "code_bleu": 0.668740304976422, + "edit_similarity": 0.75 + }, + { + "repo": "pyamg/pyamg", + "expected": "result.data)", + "got": "result.data)", + "got_raw": "result.data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "1.2 * conv_sa", + "got": "conv_sa", + "got_raw": "conv_sa", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.7 + }, + { + "repo": "pyamg/pyamg", + "expected": "1.3 * conv_sa", + "got": "conv_sa", + "got_raw": "conv_sa", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.7 + }, + { + "repo": "pyamg/pyamg", + "expected": "exact.indptr)", + "got": "exact.indptr)", + "got_raw": "exact.indptr)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "expected.nnz)", + "got": "expected.nnz)", + "got_raw": "expected.nnz)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "exact.indices)", + "got": "exact.indices)", + "got_raw": "exact.indices)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "expected.data)", + "got": "expected.data)", + "got_raw": "expected.data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "residuals[-1])", + "got": "0.0)", + "got_raw": "0.0)", + "exact_match": false, + "code_bleu": 0.17909498863725698, + "edit_similarity": 0.1111111111111111 + }, + { + "repo": "pyamg/pyamg", + "expected": "result.indptr)", + "got": "result.indptr)", + "got_raw": "result.indptr)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "result.indices)", + "got": "result.indices)", + "got_raw": "result.indices)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "rho_scale * last_rho", + "got": "rho_scale * last_rho", + "got_raw": "rho_scale * last_rho", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "rho_scale * last_rho", + "got": "rho_scale * last_rho", + "got_raw": "rho_scale * last_rho", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "Q @ coarse_candidates)", + "got": "coarse_candidates)", + "got_raw": "coarse_candidates)", + "exact_match": false, + "code_bleu": 0.3535533905932738, + "edit_similarity": 0.9 + }, + { + "repo": "pyamg/pyamg", + "expected": "expected_Cpts.shape[0])", + "got": "expected_Cpts.shape[0])", + "got_raw": "expected_Cpts.shape[0])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "len(r2)", + "got": "len(r2)", + "got_raw": "len(r2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "len(r2)", + "got": "len(r2)", + "got_raw": "len(r2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "A.todense())", + "got": "A)", + "got_raw": "A)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "pyamg/pyamg", + "expected": "x2/norm(x2))", + "got": "x2/norm(x2))", + "got_raw": "x2/norm(x2))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "x3/norm(x3))", + "got": "x/norm(x))", + "got_raw": "x/norm(x))", + "exact_match": false, + "code_bleu": 0.4111336169005197, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "pyamg/pyamg", + "expected": "np.array([]))", + "got": "np.zeros_like(diff.data))", + "got_raw": "np.zeros_like(diff.data))", + "exact_match": false, + "code_bleu": 0.24446151121745052, + "edit_similarity": 0.3684210526315789 + }, + { + "repo": "pyamg/pyamg", + "expected": "linalg.norm(A))", + "got": "np.linalg.norm(A))", + "got_raw": "np.linalg.norm(A))", + "exact_match": false, + "code_bleu": 0.7598356856515925, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "pyamg/pyamg", + "expected": "np.dot(x, P @ y))", + "got": "np.dot(x, P @ y))", + "got_raw": "np.dot(x, P @ y))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "np.linalg.norm(b)", + "got": "1.0", + "got_raw": "1.0", + "exact_match": false, + "code_bleu": 0.0625, + "edit_similarity": 0.1 + }, + { + "repo": "pyamg/pyamg", + "expected": "np.array([4, 5]))", + "got": "np.array([[0, 1], [1, 2]]))", + "got_raw": "np.array([[0, 1], [1, 2]]))", + "exact_match": false, + "code_bleu": 0.30826276460621843, + "edit_similarity": 0.6818181818181818 + }, + { + "repo": "pyamg/pyamg", + "expected": "expected.toarray())", + "got": "expected.toarray())", + "got_raw": "expected.toarray())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "resultl2.toarray())", + "got": "resultl2.toarray())", + "got_raw": "resultl2.toarray())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "expected.toarray())", + "got": "expected.toarray())", + "got_raw": "expected.toarray())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "np.sqrt(h**2 + h**2))", + "got": "h)", + "got_raw": "h)", + "exact_match": false, + "code_bleu": 0.07928047433351473, + "edit_similarity": 0.17391304347826086 + }, + { + "repo": "pyamg/pyamg", + "expected": "np.array([[3], [3]]))", + "got": "np.array([[0, 1, 2, 3, 4, 5], [0, 1, 2, 3, 4, 5]]))", + "got_raw": "np.array([[0, 1, 2, 3, 4, 5], [0, 1, 2, 3, 4, 5]]))", + "exact_match": false, + "code_bleu": 0.2656454087751724, + "edit_similarity": 0.5833333333333334 + }, + { + "repo": "pyamg/pyamg", + "expected": "np.ones(u[id3].shape))", + "got": "[1.0", + "got_raw": "[1.0, 1.0])", + "exact_match": false, + "code_bleu": 0.08954749431862849, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "pyamg/pyamg", + "expected": "1e-8*np.linalg.norm(b)", + "got": "1e-8*np.linalg.norm(b)", + "got_raw": "1e-8*np.linalg.norm(b)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "1e-8*precon_norm(b, ml)", + "got": "1e-8", + "got_raw": "1e-8", + "exact_match": false, + "code_bleu": 0.25226892457611433, + "edit_similarity": 0.2962962962962963 + }, + { + "repo": "pyamg/pyamg", + "expected": "nonsymm_lvl.A.toarray())", + "got": "symm_lvl['strength']", + "got_raw": "symm_lvl['strength'], nonsymm_lvl['strength'])", + "exact_match": false, + "code_bleu": 0.1510687698678384, + "edit_similarity": 0.45454545454545453 + }, + { + "repo": "pyamg/pyamg", + "expected": "nonsymm_lvl.A.toarray())", + "got": "symm_lvl['A']", + "got_raw": "symm_lvl['A'], nonsymm_lvl['A'])", + "exact_match": false, + "code_bleu": 0.1510687698678384, + "edit_similarity": 0.4864864864864865 + }, + { + "repo": "pyamg/pyamg", + "expected": "residuals2/norm(residuals2))", + "got": "residuals2", + "got_raw": "residuals2, residuals3)", + "exact_match": false, + "code_bleu": 0.08494336535733721, + "edit_similarity": 0.5263157894736842 + }, + { + "repo": "pyamg/pyamg", + "expected": "True)", + "got": "", + "got_raw": ", ml_filter.levels[0].P.nnz)", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "True)", + "got": "xNew.shape[0]", + "got_raw": "xNew.shape[0], x0.shape[0])", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.1111111111111111 + }, + { + "repo": "pyamg/pyamg", + "expected": "True)", + "got": "True)", + "got_raw": "True)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "False)", + "got": "False)", + "got_raw": "False)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "[0])", + "got": "np.array([0]))", + "got_raw": "np.array([0]))", + "exact_match": false, + "code_bleu": 0.392814650900513, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "pyamg/pyamg", + "expected": "[-4])", + "got": "A)", + "got_raw": "A)", + "exact_match": false, + "code_bleu": 0.2149139863647084, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "pyamg/pyamg", + "expected": "[1, 1])", + "got": "np.array([0, 1]))", + "got_raw": "np.array([0, 1]))", + "exact_match": false, + "code_bleu": 0.33180774028439425, + "edit_similarity": 0.5 + }, + { + "repo": "pyamg/pyamg", + "expected": "[1, 5])", + "got": "[1.5, 3.5])", + "got_raw": "[1.5, 3.5])", + "exact_match": false, + "code_bleu": 0.3303164318013807, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "pyamg/pyamg", + "expected": "[4, 4])", + "got": "[1, 1, 1, 1, 1, 1, 1, 1])", + "got_raw": "[1, 1, 1, 1, 1, 1, 1, 1])", + "exact_match": false, + "code_bleu": 0.101824256461955, + "edit_similarity": 0.3125 + }, + { + "repo": "pyamg/pyamg", + "expected": "[1, 6])", + "got": "[1, 4])", + "got_raw": "[1, 4])", + "exact_match": false, + "code_bleu": 0.488923022434901, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "pyamg/pyamg", + "expected": "[4, 6])", + "got": "[2, 3])", + "got_raw": "[2, 3])", + "exact_match": false, + "code_bleu": 0.3303164318013807, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "pyamg/pyamg", + "expected": "[1, 4])", + "got": "np.array([1, 4]))", + "got_raw": "np.array([1, 4]))", + "exact_match": false, + "code_bleu": 0.5156626918239822, + "edit_similarity": 0.5833333333333334 + }, + { + "repo": "pyamg/pyamg", + "expected": "[72, 9])", + "got": "np.array([72, 9]))", + "got_raw": "np.array([72, 9]))", + "exact_match": false, + "code_bleu": 0.5156626918239822, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "pyamg/pyamg", + "expected": "[15, 0])", + "got": "[15, 0])", + "got_raw": "[15, 0])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "(239, 1))", + "got": "(239, 239))", + "got_raw": "(239, 239))", + "exact_match": false, + "code_bleu": 0.488923022434901, + "edit_similarity": 0.8 + }, + { + "repo": "pyamg/pyamg", + "expected": "(240, 3))", + "got": "(239, 2))", + "got_raw": "(239, 2))", + "exact_match": false, + "code_bleu": 0.3303164318013807, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pyamg/pyamg", + "expected": "(480, 3))", + "got": "(239, 2))", + "got_raw": "(239, 2))", + "exact_match": false, + "code_bleu": 0.3303164318013807, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "pyamg/pyamg", + "expected": "[0, 0, 0])", + "got": "[0, 1, 2])", + "got_raw": "[0, 1, 2])", + "exact_match": false, + "code_bleu": 0.3688939732334406, + "edit_similarity": 0.8 + }, + { + "repo": "pyamg/pyamg", + "expected": "[0, 1, 2])", + "got": "[0, 1, 2])", + "got_raw": "[0, 1, 2])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "[1, 5, 8])", + "got": "[1.5, 7.5, 8.5])", + "got_raw": "[1.5, 7.5, 8.5])", + "exact_match": false, + "code_bleu": 0.25098621243978964, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "pyamg/pyamg", + "expected": "[4, 4, 1])", + "got": "[1, 2, 3, 4, 5, 6, 7, 8, 9])", + "got_raw": "[1, 2, 3, 4, 5, 6, 7, 8, 9])", + "exact_match": false, + "code_bleu": 0.14528679532351443, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "pyamg/pyamg", + "expected": "{i-1, i+1}", + "got": "set(A.indices[i+1:i+10])", + "got_raw": "set(A.indices[i+1:i+10])", + "exact_match": false, + "code_bleu": 0.16188613565728213, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "pyamg/pyamg", + "expected": "(239, 239))", + "got": "(vertices.shape[0], vertices.shape[0]))", + "got_raw": "(vertices.shape[0], vertices.shape[0]))", + "exact_match": false, + "code_bleu": 0.11502783619900048, + "edit_similarity": 0.2 + }, + { + "repo": "pyamg/pyamg", + "expected": "(h_split.shape[0]+1)/2", + "got": "(h_split.shape[0]+1)/2", + "got_raw": "(h_split.shape[0]+1)/2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "(h_split.shape[0]-1)/2", + "got": "(h_split.shape[0]-1)/2", + "got_raw": "(h_split.shape[0]-1)/2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "(c_split.shape[0]+1)/2", + "got": "(c_split.shape[0]+1)/2", + "got_raw": "(c_split.shape[0]+1)/2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "(c_split.shape[0]-1)/2", + "got": "(c_split.shape[0]-1)/2", + "got_raw": "(c_split.shape[0]-1)/2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyamg/pyamg", + "expected": "[0, 0, 0, 0, 1, 1, 1, 1])", + "got": "np.array([2, 3]))", + "got_raw": "np.array([2, 3]))", + "exact_match": false, + "code_bleu": 0.10409169245224004, + "edit_similarity": 0.23809523809523808 + }, + { + "repo": "pyamg/pyamg", + "expected": "[1, 0, 1, 2, 1, 0, 1, 2])", + "got": "[0, 0, 0, 0, 0, 0, 0, 0])", + "got_raw": "[0, 0, 0, 0, 0, 0, 0, 0])", + "exact_match": false, + "code_bleu": 0.2239521374625909, + "edit_similarity": 0.6 + }, + { + "repo": "pyamg/pyamg", + "expected": "[0, 0, 0, 0, 1, 1, 1, 1, 2])", + "got": "np.array([2, 2, 2]))", + "got_raw": "np.array([2, 2, 2]))", + "exact_match": false, + "code_bleu": 0.18796602066011717, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "pyamg/pyamg", + "expected": "[[0, 3, 2], [0, 1, 3]])", + "got": "[[0, 1, 2],", + "got_raw": "[[0, 1, 2],", + "exact_match": false, + "code_bleu": 0.4445698525097307, + "edit_similarity": 0.5294117647058824 + }, + { + "repo": "pyamg/pyamg", + "expected": "[[0., 0.], [1., 0.], [0., 1.], [1., 1.]])", + "got": "[[0.0, 0.0], [1.0, 0.0], [0.5, 0.5]])", + "got_raw": "[[0.0, 0.0], [1.0, 0.0], [0.5, 0.5]])", + "exact_match": false, + "code_bleu": 0.18494734785944192, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "pyamg/pyamg", + "expected": "[[0, 4, 3], [1, 5, 4], [0, 1, 4], [1, 2, 5]])", + "got": "[[0, 1, 2],", + "got_raw": "[[0, 1, 2],", + "exact_match": false, + "code_bleu": 0.2079606975355657, + "edit_similarity": 0.35714285714285715 + }, + { + "repo": "pyamg/pyamg", + "expected": "[[0., 0.], [0.5, 0.], [1., 0.], [0., 1.], [0.5, 1.], [1., 1.]])", + "got": "[[0.0, 0.0], [1.0, 0.0], [0.5, 0.8660254037844386], [0.5, -0.8660254037844386]])", + "got_raw": "[[0.0, 0.0], [1.0, 0.0], [0.5, 0.8660254037844386], [0.5, -0.8660254037844386]])", + "exact_match": false, + "code_bleu": 0.2878136191093911, + "edit_similarity": 0.4755244755244755 + }, + { + "repo": "pyamg/pyamg", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "9", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "torch_frame.stype('numerical')", + "got": "torch_frame.stype[\"int\"]", + "got_raw": "torch_frame.stype[\"int\"]", + "exact_match": false, + "code_bleu": 0.4111336169005197, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "torch_frame.stype('embedding')", + "got": "torch_frame.stype('embedding')", + "got_raw": "torch_frame.stype('embedding')", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "torch_frame.stype('categorical')", + "got": "torch_frame.stype('categorical')", + "got_raw": "torch_frame.stype('categorical')", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "torch_frame.stype('text_embedded')", + "got": "torch_frame.stype('text_embedded')", + "got_raw": "torch_frame.stype('text_embedded')", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "1", + "got": "5", + "got_raw": "5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "[]", + "got": "['target']", + "got_raw": "['target']", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "[4, 3, 1]", + "got": "[2, 1, 1]", + "got_raw": "[2, 1, 1]", + "exact_match": false, + "code_bleu": 0.38260294162784475, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "task_type", + "got": "task_type", + "got_raw": "task_type", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "num_classes", + "got": "num_classes", + "got_raw": "num_classes", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "len(dataset)", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "([1, 0], [3, 1])", + "got": "4", + "got_raw": "4", + "exact_match": false, + "code_bleu": 0.038461538461538464, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "([0, 1], [1, 3])", + "got": "([1, 1, 1, 1], [4, 4, 4, 4])", + "got_raw": "([1, 1, 1, 1], [4, 4, 4, 4])", + "exact_match": false, + "code_bleu": 0.309936240366613, + "edit_similarity": 0.5909090909090909 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "['cat_1', 'cat_2']", + "got": "_frame.col_types_dict[torch_frame.numerical]", + "got_raw": "_frame.col_types_dict[torch_frame.numerical]", + "exact_match": false, + "code_bleu": 0.17747405280050269, + "edit_similarity": 0.12903225806451613 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "test_dataset.col_stats", + "got": "test_dataset.col_stats", + "got_raw": "test_dataset.col_stats", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "['num_1', 'num_2', 'num_3']", + "got": "ortframe()", + "got_raw": "ortframe()", + "exact_match": false, + "code_bleu": 0.16282193263962697, + "edit_similarity": 0.05405405405405406 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "''", + "got": "''", + "got_raw": "''", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "'B'", + "got": "'B'", + "got_raw": "'B'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "ser)", + "got": "ser)", + "got_raw": "ser)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(2, 7)", + "got": "(2, 1)", + "got_raw": "(2, 1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "torch.long", + "got": "torch.long", + "got_raw": "torch.long", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "torch.float", + "got": "torch.float32", + "got_raw": "torch.float32", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.9166666666666666 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "torch.float32", + "got": "torch.float32", + "got_raw": "torch.float32", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(num_sentences + 1, 1, -1)", + "got": "(num_sentences, out_channels)", + "got_raw": "(num_sentences, out_channels)", + "exact_match": false, + "code_bleu": 0.2041241452319315, + "edit_similarity": 0.6181818181818182 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(num_sentences + 1, out_channels)", + "got": "(num_sentences + 1, 1, out_channels)", + "got_raw": "(num_sentences + 1, 1, out_channels)", + "exact_match": false, + "code_bleu": 0.7186082239261685, + "edit_similarity": 0.9565217391304348 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "pd.Series(['A', 'B', None, None, 'B']))", + "got": "ser)", + "got_raw": "ser)", + "exact_match": false, + "code_bleu": 0.06320999598962011, + "edit_similarity": 0.13953488372093023 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "pd.Series([[0.1, 0.5], [0.3], None, [0.2, np.nan], None, None]))", + "got": "(ser, out)", + "got_raw": "(ser, out)", + "exact_match": false, + "code_bleu": 0.05236579281856434, + "edit_similarity": 0.16216216216216217 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "1", + "got": "num_rows", + "got_raw": "num_rows", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "-1", + "got": "lengths.sum().item()", + "got_raw": "lengths.sum().item()", + "exact_match": false, + "code_bleu": 0.11868405219520975, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "met)", + "got": "met)", + "got_raw": "met)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "12345.", + "got": "12345", + "got_raw": "12345", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "met[:])", + "got": "met[0:2])", + "got_raw": "met[0:2])", + "exact_match": false, + "code_bleu": 0.32172944208038085, + "edit_similarity": 0.875 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "num_rows", + "got": "num_rows", + "got_raw": "num_rows", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "num_cols", + "got": "num_cols", + "got_raw": "num_cols", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "met[:3])", + "got": "met[:3])", + "got_raw": "met[:3])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "met[3:])", + "got": "met[3:])", + "got_raw": "met[3:])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "1", + "got": "len(tensor_mat[i])", + "got_raw": "len(tensor_mat[i])", + "exact_match": false, + "code_bleu": 0.1561969968460128, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "multi_nested_tensor.shape[0]", + "got": "len(multi_nested_tensor)", + "got_raw": "len(multi_nested_tensor)", + "exact_match": false, + "code_bleu": 0.23953607464205617, + "edit_similarity": 0.7307692307692307 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "multi_nested_tensor.shape[1]", + "got": "multi_nested_tensor.shape[1]", + "got_raw": "multi_nested_tensor.shape[1]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "RuntimeError, match=\"The length of each row must be the same\")", + "got": "", + "got_raw": "", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "year_range[1]", + "got": "1", + "got_raw": "1 and year_range[1]", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "expected_col_stats", + "got": "expected_col_stats", + "got_raw": "expected_col_stats", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "TypeError, match='Numerical series contains invalid entries.')", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.17909498863725698, + "edit_similarity": 0.1917808219178082 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "{ StatType.EMB_DIM: emb_dim, }", + "got": "{StatType.MEAN: np.zeros(emb_dim)}", + "got_raw": "{StatType.MEAN: np.zeros(emb_dim)}", + "exact_match": false, + "code_bleu": 0.2224246939793677, + "edit_similarity": 0.65625 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "{ StatType.COUNT: (['a', 'b', 'c'], [3, 1, 1]), }", + "got": "{StatType.COUNT: 5, StatType.FREQUENCY: {'a': 3, 'b': 1, 'c': 1}}", + "got_raw": "{StatType.COUNT: 5, StatType.FREQUENCY: {'a': 3, 'b': 1, 'c': 1}}", + "exact_match": false, + "code_bleu": 0.26343460637499544, + "edit_similarity": 0.5789473684210527 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "{ StatType.MULTI_COUNT: (['a', 'c', 'b'], [4, 3, 2]), }", + "got": "{", + "got_raw": "{", + "exact_match": false, + "code_bleu": 0.024775148229223352, + "edit_similarity": 0.03571428571428571 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "{ StatType.MEAN: 2.0, StatType.STD: 0.816496580927726, StatType.QUANTILES: [1.0, 1.5, 2.0, 2.5, 3.0], }", + "got": "{StatType.MEAN: 2.0}", + "got_raw": "{StatType.MEAN: 2.0}", + "exact_match": false, + "code_bleu": 0.202869740240996, + "edit_similarity": 0.3252032520325203 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "{ StatType.MEAN: 3.5, StatType.STD: 1.707825127659933, StatType.QUANTILES: [1.0, 2.25, 3.5, 4.75, 6.0], }", + "got": "{StatType.MEAN: 3.0, StatType.STD: 2.0}", + "got_raw": "{StatType.MEAN: 3.0, StatType.STD: 2.0}", + "exact_match": false, + "code_bleu": 0.2729189663370838, + "edit_similarity": 0.5277777777777778 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "{ StatType.MEAN: np.nan, StatType.STD: np.nan, StatType.QUANTILES: [np.nan, np.nan, np.nan, np.nan, np.nan], }", + "got": "{StatType.MEAN: np.nan}", + "got_raw": "{StatType.MEAN: np.nan}", + "exact_match": false, + "code_bleu": 0.18482768255655918, + "edit_similarity": 0.3458646616541353 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "16", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "tf2", + "got": "tf2", + "got_raw": "tf2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "tf1", + "got": "tf1", + "got_raw": "tf1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "tf)", + "got": "tf", + "got_raw": "tf, False)", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.8 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "None", + "got": "torch.device(\"cpu\")", + "got_raw": "torch.device(\"cpu\")", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.17391304347826086 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "8124", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "('Dota2', {})", + "got": "('AdultCensusIncome', {})", + "got_raw": "('AdultCensusIncome', {})", + "exact_match": false, + "code_bleu": 0.6434588841607617, + "edit_similarity": 0.47368421052631576 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "('Mushroom', {})", + "got": "('AdultCensusIncome', {'scale': 'medium'})", + "got_raw": "('AdultCensusIncome', {'scale': 'medium'})", + "exact_match": false, + "code_bleu": 0.24446151121745052, + "edit_similarity": 0.41379310344827586 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "('PokerHand', {})", + "got": "('TabularBenchmark', {'name': 'Diabetes130US'})", + "got_raw": "('TabularBenchmark', {'name': 'Diabetes130US'})", + "exact_match": false, + "code_bleu": 0.24446151121745052, + "edit_similarity": 0.3125 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "('BankMarketing', {})", + "got": "('AdultCensusIncome', {'scale': 'medium'})", + "got_raw": "('AdultCensusIncome', {'scale': 'medium'})", + "exact_match": false, + "code_bleu": 0.24446151121745052, + "edit_similarity": 0.31746031746031744 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "\"DataFrameBenchmark(\\n\" \" task_type=binary_classification,\\n\" \" scale=small,\\n\" \" idx=1,\\n\" \" cls=Mushroom()\\n\" \")\"", + "got": "f\"DataFrameBenchmark(task_type='binary_classification', scale='small', idx=1)\")", + "got_raw": "f\"DataFrameBenchmark(task_type='binary_classification', scale='small', idx=1)\")", + "exact_match": false, + "code_bleu": 0.18995892141289814, + "edit_similarity": 0.7525773195876289 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "0", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "6364", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "5091", + "got": "dataset.num_rows", + "got_raw": "dataset.num_rows", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "1273", + "got": "225", + "got_raw": "225", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "('Mercari', {})", + "got": "('HuggingFaceDatasetDict', {", + "got_raw": "('HuggingFaceDatasetDict', {", + "exact_match": false, + "code_bleu": 0.3387551654364098, + "edit_similarity": 0.32558139534883723 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "('AmazonFineFoodReviews', {})", + "got": "('HuggingFaceDatasetDict', {", + "got_raw": "('HuggingFaceDatasetDict', {", + "exact_match": false, + "code_bleu": 0.3387551654364098, + "edit_similarity": 0.38596491228070173 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "('MultimodalTextBenchmark', { 'name': 'news_channel' })", + "got": "('MultimodalTextBenchmark', {", + "got_raw": "('MultimodalTextBenchmark', {", + "exact_match": false, + "code_bleu": 0.4444444444444444, + "edit_similarity": 0.6904761904761905 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "('MultimodalTextBenchmark', { 'name': 'wine_reviews' })", + "got": "('MultimodalTextBenchmark', {", + "got_raw": "('MultimodalTextBenchmark', {", + "exact_match": false, + "code_bleu": 0.4444444444444444, + "edit_similarity": 0.6904761904761905 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "\"DataFrameTextBenchmark(\\n\" \" task_type=multiclass_classification,\\n\" \" scale=small,\\n\" \" idx=0,\\n\" \" cls=MultimodalTextBenchmark()\\n\" \")\"", + "got": "f\"DataFrameTextBenchmark(task_type='multiclass_classification', scale='small', idx=0, col_to_text_embedder_cfg=TextEmbedderConfig(text_embedder=HashTextEmbedder(10)))\")", + "got_raw": "f\"DataFrameTextBenchmark(task_type='multiclass_classification', scale='small', idx=0, col_to_text_embedder_cfg=TextEmbedderConfig(text_embedder=HashTextEmbedder(10)))\")", + "exact_match": false, + "code_bleu": 0.18995892141289814, + "edit_similarity": 0.6078431372549019 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "num_rows", + "got": "num_rows", + "got_raw": "num_rows", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "4", + "got": "7", + "got_raw": "7", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "10", + "got": "7", + "got_raw": "7", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "1000209", + "got": "10000", + "got_raw": "10000", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "torch.int64", + "got": "torch.int64", + "got_raw": "torch.int64", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "2", + "got": "7", + "got_raw": "7", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "8", + "got": "7", + "got_raw": "7", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "891", + "got": "891", + "got_raw": "891", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(891, )", + "got": "(891, 1)", + "got_raw": "(891, 1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(891, 4)", + "got": "(6, 891)", + "got_raw": "(6, 891)", + "exact_match": false, + "code_bleu": 0.34329452398451965, + "edit_similarity": 0.625 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "0", + "got": "0.0).all()", + "got_raw": "0.0).all()", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "score", + "got": "score)", + "got_raw": "score)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "metric", + "got": "loaded_gbdt.metric", + "got_raw": "loaded_gbdt.metric", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.5 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(10, 8)", + "got": "(10, 4)", + "got_raw": "(10, 4)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(10, 4, 8)", + "got": "(10, 2, 16)", + "got_raw": "(10, 2, 16)", + "exact_match": false, + "code_bleu": 0.38260294162784475, + "edit_similarity": 0.7619047619047619 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(10, 4, 16)", + "got": "(10, 4, 16)", + "got_raw": "(10, 4, 16)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "['num1', 'num2', 'cat1', 'cat2']", + "got": "['num1', 'num2', 'cat1', 'cat2']", + "got_raw": "['num1', 'num2', 'cat1', 'cat2']", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(batch_size, num_cols, channels)", + "got": "(batch_size, num_cols, num_heads, channels // num_heads)", + "got_raw": "(batch_size, num_cols, num_heads, channels // num_heads)", + "exact_match": false, + "code_bleu": 0.42803206067505944, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(10, 8)", + "got": "(10, 1, 8)", + "got_raw": "(10, 1, 8)", + "exact_match": false, + "code_bleu": 0.48109772909788073, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(10, 3, 8)", + "got": "(10, 4, 8)", + "got_raw": "(10, 4, 8)", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(batch_size, num_cols, channels)", + "got": "(batch_size, num_cols, num_heads, channels // num_heads)", + "got_raw": "(batch_size, num_cols, num_heads, channels // num_heads)", + "exact_match": false, + "code_bleu": 0.42803206067505944, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(batch_size, num_prompts, channels)", + "got": "(batch_size, num_cols, num_prompts, 8)", + "got_raw": "(batch_size, num_cols, num_prompts, 8)", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.7397260273972602 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(batch_size, out_channels)", + "got": "(batch_size, out_channels)", + "got_raw": "(batch_size, out_channels)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(batch_size, out_channels)", + "got": "(batch_size, num_prompts, out_channels)", + "got_raw": "(batch_size, num_prompts, out_channels)", + "exact_match": false, + "code_bleu": 0.48109772909788073, + "edit_similarity": 0.8 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "len(dataset.df.dropna())", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.05, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(feat_cat.size(0), feat_cat.size(1), 8)", + "got": "(10, 8)", + "got_raw": "(10, 8)", + "exact_match": false, + "code_bleu": 0.1580249899740503, + "edit_similarity": 0.2608695652173913 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(feat_num.size(0), feat_num.size(1), 8)", + "got": "(10, 8)", + "got_raw": "(10, 8)", + "exact_match": false, + "code_bleu": 0.1580249899740503, + "edit_similarity": 0.2608695652173913 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(feat_multicat.size(0), feat_multicat.size(1), 8)", + "got": "(10, 8)", + "got_raw": "(10, 8)", + "exact_match": false, + "code_bleu": 0.1580249899740503, + "edit_similarity": 0.21428571428571427 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(feat_timestamp.size(0), feat_timestamp.size(1), 8)", + "got": "(torch.Size([10, 8]), torch.Size([10, 8]))", + "got_raw": "(torch.Size([10, 8]), torch.Size([10, 8]))", + "exact_match": false, + "code_bleu": 0.1090009697802911, + "edit_similarity": 0.4731182795698925 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "tensor_frame.feat_dict[ stype.text_tokenized].keys()", + "got": "col_names", + "got_raw": "col_names", + "exact_match": false, + "code_bleu": 0.041666666666666664, + "edit_similarity": 0.19672131147540983 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "( num_rows, len(tensor_frame.col_names_dict[data_stype]), out_channels, )", + "got": "(col_names.shape[0], out_channels)", + "got_raw": "(col_names.shape[0], out_channels)", + "exact_match": false, + "code_bleu": 0.1325039172439074, + "edit_similarity": 0.5420560747663551 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "( num_rows, len(tensor_frame.col_names_dict[stype.embedding]), out_channels, )", + "got": "(torch.Size([num_rows, out_channels]),)", + "got_raw": "(torch.Size([num_rows, out_channels]),)", + "exact_match": false, + "code_bleu": 0.20095561263810596, + "edit_similarity": 0.41025641025641024 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "( num_rows, len(tensor_frame.col_names_dict[stype.text_tokenized]), out_channels, )", + "got": "(num_rows, out_channels)", + "got_raw": "(num_rows, out_channels)", + "exact_match": false, + "code_bleu": 0.16785418874204266, + "edit_similarity": 0.4485981308411215 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "tensor_frame", + "got": "tensor_frame", + "got_raw": "tensor_frame", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "ValueError, match=\"is an invalid stype\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.43137254901960786 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(num_rows, tensor_frame.num_cols, out_channels)", + "got": "(col_names.shape[0], out_channels)", + "got_raw": "(col_names.shape[0], out_channels)", + "exact_match": false, + "code_bleu": 0.25965358893403384, + "edit_similarity": 0.49382716049382713 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "[ \"num_1\", \"num_2\", \"num_3\", \"cat_1\", \"cat_2\", \"text_tokenized_1\", \"text_tokenized_2\", \"multicat_1\", \"multicat_2\", \"multicat_3\", \"multicat_4\", \"timestamp_0\", \"timestamp_1\", \"timestamp_2\", \"emb_1\", \"emb_2\", \"text_embedded_1\", \"text_embedded_2\", ]", + "got": "col_names", + "got_raw": "col_names", + "exact_match": false, + "code_bleu": 0.013157894736842105, + "edit_similarity": 0.031496062992125984 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "input_tensor.shape + (out_size, )", + "got": "(*size[:-1], out_size)", + "got_raw": "(*size[:-1], out_size)", + "exact_match": false, + "code_bleu": 0.14323145079400493, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "input_tensor.shape + (out_size, )", + "got": "(size + (out_size, ))", + "got_raw": "(size + (out_size, ))", + "exact_match": false, + "code_bleu": 0.5873949094699213, + "edit_similarity": 0.6296296296296297 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "expected_graph_breaks", + "got": "expected_graph_breaks", + "got_raw": "expected_graph_breaks", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "tensor_frame.y.shape", + "got": "(batch_size", + "got_raw": "(batch_size, 1)", + "exact_match": false, + "code_bleu": 0.18072040072196896, + "edit_similarity": 0.1935483870967742 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(batch_size, out_channels)", + "got": "(batch_size, out_channels)", + "got_raw": "(batch_size, out_channels)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(batch_size, out_channels)", + "got": "(batch_size, out_channels)", + "got_raw": "(batch_size, out_channels)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(batch_size, out_channels)", + "got": "(batch_size, out_channels)", + "got_raw": "(batch_size, out_channels)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(batch_size, out_channels)", + "got": "(batch_size, out_channels)", + "got_raw": "(batch_size, out_channels)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(batch_size, out_channels)", + "got": "(batch_size, out_channels)", + "got_raw": "(batch_size, out_channels)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "0", + "got": "0.0", + "got_raw": "0.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(len(tensor_frame), out_channels)", + "got": "(batch_size, out_channels)", + "got_raw": "(batch_size, out_channels)", + "exact_match": false, + "code_bleu": 0.3358031036948569, + "edit_similarity": 0.6101694915254238 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "(batch_size, num_layers, out_channels)", + "got": "(batch_size, num_prompts, out_channels)", + "got_raw": "(batch_size, num_prompts, out_channels)", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8831168831168831 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "col_stats", + "got": "col_stats)", + "got_raw": "col_stats)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "total_cols", + "got": "total_numerical_cols)", + "got_raw": "total_numerical_cols)", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.6451612903225806 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "out.feat_dict", + "got": "out.col_types_dict)", + "got_raw": "out.col_types_dict)", + "exact_match": false, + "code_bleu": 0.47287080450158786, + "edit_similarity": 0.625 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "out.col_names_dict", + "got": "out.col_names_dict)", + "got_raw": "out.col_names_dict)", + "exact_match": false, + "code_bleu": 0.668740304976422, + "edit_similarity": 0.972972972972973 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "out.feat_dict.keys()", + "got": "loaded_out.col_names_dict.keys())", + "got_raw": "loaded_out.col_names_dict.keys())", + "exact_match": false, + "code_bleu": 0.46713797772820004, + "edit_similarity": 0.6415094339622641 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "len(categorical_features)", + "got": "len(categorical_features))", + "got_raw": "len(categorical_features))", + "exact_match": false, + "code_bleu": 0.7598356856515925, + "edit_similarity": 0.9803921568627451 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "out.col_names_dict.keys()", + "got": "loaded_out.col_names_dict)", + "got_raw": "loaded_out.col_names_dict)", + "exact_match": false, + "code_bleu": 0.29036157037406557, + "edit_similarity": 0.7450980392156863 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "loaded_out.col_names_dict", + "got": "transform.col_names_dict)", + "got_raw": "transform.col_names_dict)", + "exact_match": false, + "code_bleu": 0.47287080450158786, + "edit_similarity": 0.64 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "transformed_col_stats[col]", + "got": "col)", + "got_raw": "col)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.2 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "'num_1'", + "got": "'num_1')", + "got_raw": "'num_1')", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "tensor_frame.feat_dict[ stype.numerical].size()", + "got": "tensor_frame.feat_dict[", + "got_raw": "tensor_frame.feat_dict[", + "exact_match": false, + "code_bleu": 0.3333333333333333, + "edit_similarity": 0.6571428571428571 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "set( tensor_frame.col_names_dict[stype.numerical])", + "got": "set(dataset.col_names))", + "got_raw": "set(dataset.col_names))", + "exact_match": false, + "code_bleu": 0.17675757489165944, + "edit_similarity": 0.4657534246575342 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "tf", + "got": "tf", + "got_raw": "tf", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "num_rows * num_repeats", + "got": "num_rows * num_repeats", + "got_raw": "num_rows * num_repeats", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "dataset.col_to_stype", + "got": "dataset.col_to_stype", + "got_raw": "dataset.col_to_stype", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "torch_frame.categorical", + "got": "torch_frame.boolean", + "got_raw": "torch_frame.boolean", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "torch_frame.multicategorical", + "got": "torch_frame.multicategorical", + "got_raw": "torch_frame.multicategorical", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "tf", + "got": "tf", + "got_raw": "tf", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "col_stats", + "got": "col_stats", + "got_raw": "col_stats", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "new_dataset.col_stats", + "got": "new_dataset.col_stats", + "got_raw": "new_dataset.col_stats", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "new_dataset.tensor_frame", + "got": "new_dataset.tensor_frame", + "got_raw": "new_dataset.tensor_frame", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "4 * 8 * 4", + "got": "32", + "got_raw": "32", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "12 * 4 + 7 * 8", + "got": "12 * 4", + "got_raw": "12 * 4", + "exact_match": false, + "code_bleu": 0.36038417796587763, + "edit_similarity": 0.6 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "2 * 10 * 4 + 4 * 8", + "got": "2 * 10 * 4", + "got_raw": "2 * 10 * 4", + "exact_match": false, + "code_bleu": 0.5555555555555556, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "2 * num_bytes(data)", + "got": "2 * (2 * 10 * 4 + 4 * 8)", + "got_raw": "2 * (2 * 10 * 4 + 4 * 8)", + "exact_match": false, + "code_bleu": 0.14283632578659286, + "edit_similarity": 0.27906976744186046 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "3 * num_bytes(data)", + "got": "3 * num_bytes(data)", + "got_raw": "3 * num_bytes(data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "int(num_data * val_ratio)", + "got": "int(num_data * val_ratio)", + "got_raw": "int(num_data * val_ratio)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "int(num_data * test_ratio)", + "got": "int(num_data * test_ratio)", + "got_raw": "int(num_data * test_ratio)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyg-team/pytorch-frame", + "expected": "int(num_data * train_ratio)", + "got": "int(train_ratio * num_data)", + "got_raw": "int(train_ratio * num_data)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.5925925925925926 + }, + { + "repo": "pypa/pipx", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "\"\"", + "got": "ref_metadata.package_version", + "got_raw": "ref_metadata.package_version", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "pypa/pipx", + "expected": "ref_metadata_replaced", + "got": "ref_metadata_replaced", + "got_raw": "ref_metadata_replaced", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.err", + "got": "captured.err", + "got_raw": "captured.err", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "expected_string", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pypa/pipx", + "expected": "expected_string[:chars_to_test]", + "got": "expected_string", + "got_raw": "expected_string", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.6521739130434783 + }, + { + "repo": "pypa/pipx", + "expected": "exposed", + "got": "exposed", + "got_raw": "exposed", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.out", + "got": "captured.err", + "got_raw": "captured.err", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.75 + }, + { + "repo": "pypa/pipx", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pypa/pipx", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "SystemExit)", + "got": "SystemExit)", + "got_raw": "SystemExit)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.out", + "got": "captured.out", + "got_raw": "captured.out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.err", + "got": "captured.err", + "got_raw": "captured.err", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "str(env_dir)", + "got": "env_dir", + "got_raw": "env_dir", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "pypa/pipx", + "expected": "home / \"test\"", + "got": "home / \"test\"", + "got_raw": "home / \"test\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "{pkg_name}", + "got": "{pkg_name}", + "got_raw": "{pkg_name}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "caplog.text", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "capsys.readouterr().err", + "got": "capsys.read()", + "got_raw": "capsys.read()", + "exact_match": false, + "code_bleu": 0.3227150012892303, + "edit_similarity": 0.7222222222222222 + }, + { + "repo": "pypa/pipx", + "expected": "{pkg for pkg, _ in packages}", + "got": "set(pkg[0] for pkg in packages)", + "got_raw": "set(pkg[0] for pkg in packages)", + "exact_match": false, + "code_bleu": 0.1972940627795883, + "edit_similarity": 0.7796610169491526 + }, + { + "repo": "pypa/pipx", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pypa/pipx", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "caplog.text", + "got": "log.text", + "got_raw": "log.text", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.8421052631578947 + }, + { + "repo": "pypa/pipx", + "expected": "captured.out", + "got": "captured.out", + "got_raw": "captured.out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.out", + "got": "captured.out", + "got_raw": "captured.out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.err", + "got": "captured.err", + "got_raw": "captured.err", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "log_contents", + "got": "log_contents", + "got_raw": "log_contents", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "\"py\"", + "got": "\"py\"", + "got_raw": "\"py\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "str(e)", + "got": "e.value", + "got_raw": "e.value", + "exact_match": false, + "code_bleu": 0.3388507513536918, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "pypa/pipx", + "expected": "\"python\"", + "got": "\"python\"", + "got_raw": "\"python\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "PipxError)", + "got": "InterpreterResolutionError)", + "got_raw": "InterpreterResolutionError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3783783783783784 + }, + { + "repo": "pypa/pipx", + "expected": "python_path", + "got": "python_path", + "got_raw": "python_path", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "\"WindowsApps\"", + "got": "\"3.8\"", + "got_raw": "\"3.8\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "pypa/pipx", + "expected": "sys.executable", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.0 + }, + { + "repo": "pypa/pipx", + "expected": "InterpreterResolutionError)", + "got": "InterpreterResolutionError)", + "got_raw": "InterpreterResolutionError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "{}", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "pypa/pipx", + "expected": "[\"black\"]", + "got": "[\"dill\"", + "got_raw": "[\"dill\", \"isort\"]", + "exact_match": false, + "code_bleu": 0.35818997727451396, + "edit_similarity": 0.5 + }, + { + "repo": "pypa/pipx", + "expected": "captured.err", + "got": "captured.out", + "got_raw": "captured.out", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.75 + }, + { + "repo": "pypa/pipx", + "expected": "captured.out", + "got": "captured.out", + "got_raw": "captured.out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "black_package_ref)", + "got": "{", + "got_raw": "{ \"black\": PackageInfo(**black_package_ref.metadata[\"main_package\"]) })", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "pypa/pipx", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.out", + "got": "captured.out", + "got_raw": "captured.out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.out.strip()", + "got": "captured.out", + "got_raw": "captured.out", + "exact_match": false, + "code_bleu": 0.36038417796587763, + "edit_similarity": 0.75 + }, + { + "repo": "pypa/pipx", + "expected": "ValueError, match=\"raised in test to exit early\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.36666666666666664 + }, + { + "repo": "pypa/pipx", + "expected": "caplog.text", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "package_name_out", + "got": "package_name_out", + "got_raw": "package_name_out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "package_spec_out", + "got": "package_spec_out", + "got_raw": "package_spec_out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "package_or_url_correct", + "got": "package_or_url_correct", + "got_raw": "package_or_url_correct", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "PipxError, match=r\"^Unable to parse package spec\")", + "got": "PipxError)", + "got_raw": "PipxError)", + "exact_match": false, + "code_bleu": 0.16988673071467442, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "pypa/pipx", + "expected": "caplog.text", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.err", + "got": "captured.err", + "got_raw": "captured.err", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.out", + "got": "captured", + "got_raw": "captured", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.8 + }, + { + "repo": "pypa/pipx", + "expected": "{}", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "pypa/pipx", + "expected": "{\"black\"}", + "got": "{\"black\"}", + "got_raw": "{\"black\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "PipxError)", + "got": "PipxError)", + "got_raw": "PipxError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "black_package_ref)", + "got": "black_package_ref)", + "got_raw": "black_package_ref)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "pycowsay_package_ref)", + "got": "pycowsay_package_ref)", + "got_raw": "pycowsay_package_ref)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "getattr(pipx_metadata2, attribute)", + "got": "getattr(pipx_metadata2, attribute)", + "got_raw": "getattr(pipx_metadata2, attribute)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.out", + "got": "captured.out", + "got_raw": "captured.out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.err", + "got": "captured.out", + "got_raw": "captured.out", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.75 + }, + { + "repo": "pypa/pipx", + "expected": "capsys.readouterr().out", + "got": "capsys.read()", + "got_raw": "capsys.read()", + "exact_match": false, + "code_bleu": 0.3227150012892303, + "edit_similarity": 0.7222222222222222 + }, + { + "repo": "pypa/pipx", + "expected": "captured.err.replace(\"\\n\", \" \")", + "got": "captured.err", + "got_raw": "captured.err", + "exact_match": false, + "code_bleu": 0.25226892457611433, + "edit_similarity": 0.5581395348837209 + }, + { + "repo": "pypa/pipx", + "expected": "caplog.text", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.out", + "got": "captured", + "got_raw": "captured", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.8 + }, + { + "repo": "pypa/pipx", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "\"2\"", + "got": "\"2\"", + "got_raw": "\"2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "test_str", + "got": "test_str", + "got_raw": "test_str", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "SystemExit", + "got": "SystemExit", + "got_raw": "SystemExit", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "caplog.text", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "SystemExit)", + "got": "SystemExit)", + "got_raw": "SystemExit)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "needs_upgrade", + "got": "needs_upgrade", + "got_raw": "needs_upgrade", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.out", + "got": "captured.out", + "got_raw": "captured.out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "caplog.text", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.out", + "got": "captured.err", + "got_raw": "captured.err", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.75 + }, + { + "repo": "pypa/pipx", + "expected": "caplog.text", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.out", + "got": "captured.err", + "got_raw": "captured.err", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.75 + }, + { + "repo": "pypa/pipx", + "expected": "captured.err", + "got": "captured.err", + "got_raw": "captured.err", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.err", + "got": "captured.err", + "got_raw": "captured.err", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.out", + "got": "captured.out", + "got_raw": "captured.out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "caplog.text", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "captured.out", + "got": "captured.err", + "got_raw": "captured.err", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.75 + }, + { + "repo": "pypa/pipx", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pypa/pipx", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/pipx", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "help_text", + "got": "help_text", + "got_raw": "help_text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "''", + "got": "''", + "got_raw": "''", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "'tag'", + "got": "'argparse_app'", + "got_raw": "'argparse_app'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "'True'", + "got": "'we'", + "got_raw": "'we'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "out[1]", + "got": "err", + "got_raw": "err", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "\"foo\\n\"", + "got": "\"say:", + "got_raw": "\"say: word\\n\"", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "'custom'", + "got": "'{\"arg1\":", + "got_raw": "'{\"arg1\": \"arg1\", \"arg2\": \"arg2\"}'", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "['10.0']", + "got": "'foo:", + "got_raw": "'foo: 5.0\\n'", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "['hello']", + "got": "'hello\\n'", + "got_raw": "'hello\\n'", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "out2", + "got": "out2", + "got_raw": "out2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "show_table", + "got": "show_table", + "got_raw": "show_table", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "\"--myflag\"", + "got": "'mychoice'", + "got_raw": "'mychoice'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "num_aliases", + "got": "num_aliases", + "got_raw": "num_aliases", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "display_meta", + "got": "display_meta", + "got_raw": "display_meta", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "CustomCompleter", + "got": "CustomCompleter", + "got_raw": "CustomCompleter", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "completions[0].text", + "got": "completions[0]", + "got_raw": "completions[0]", + "exact_match": false, + "code_bleu": 0.6666666666666666, + "edit_similarity": 0.8484848484848485 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "\"Hint:\\n no_help_pos\"", + "got": "'foo'", + "got_raw": "'foo'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.14814814814814814 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "completions.completion_error", + "got": "completions", + "got_raw": "completions", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5641025641025641 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "len(ac_app.num_completion_items)", + "got": "len(ac_app.completion_item_choices)", + "got_raw": "len(ac_app.completion_item_choices)", + "exact_match": false, + "code_bleu": 0.6434588841607617, + "edit_similarity": 0.835820895522388 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "expected_completions.to_strings()", + "got": "expected_completions.to_strings()", + "got_raw": "expected_completions.to_strings()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "out", + "got": "out", + "got_raw": "out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "err", + "got": "err", + "got_raw": "err", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "'1'", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "'2'", + "got": "'2'", + "got_raw": "'2'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "out[0]", + "got": "out", + "got_raw": "out", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "out[1]", + "got": "out[0]", + "got_raw": "out[0]", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "err[2]", + "got": "err", + "got_raw": "err", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "(0, 3)", + "got": "(0, 3)", + "got_raw": "(0, 3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "(1, 5)", + "got": "(1, 5)", + "got_raw": "(1, 5)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "(2, 5)", + "got": "(2, 5)", + "got_raw": "(2, 5)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "new_val", + "got": "new_val", + "got_raw": "new_val", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "''", + "got": "'base'", + "got_raw": "'base'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "['10.0']", + "got": "'foo:", + "got_raw": "'foo: 5.0\\n'", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "['((baz))']", + "got": "'bar", + "got_raw": "'bar baz\\n'", + "exact_match": false, + "code_bleu": 0.30120066786994826, + "edit_similarity": 0.4 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "'Base command help'", + "got": "'base'", + "got_raw": "'base'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "'Default Category'", + "got": "\"Base\"", + "got_raw": "\"Base\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "out", + "got": "out", + "got_raw": "out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "err", + "got": "err", + "got_raw": "err", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "app", + "got": "app", + "got_raw": "app", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "matches", + "got": "matches", + "got_raw": "matches", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "5", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "'\"'", + "got": "'\"'", + "got_raw": "'\"'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "\"'\"", + "got": "\"'\"", + "got_raw": "\"'\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "items", + "got": "items", + "got_raw": "items", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "tokens", + "got": "tokens", + "got_raw": "tokens", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "values", + "got": "values", + "got_raw": "values", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "display", + "got": "display", + "got_raw": "display", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "\"Apple\"", + "got": "\"apple\"", + "got_raw": "\"apple\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "all_nums", + "got": "all_nums", + "got_raw": "all_nums", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "raw_tokens", + "got": "raw_tokens", + "got_raw": "raw_tokens", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "items[::-1]", + "got": "list(reversed(items))", + "got_raw": "list(reversed(items))", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.3125 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "completions", + "got": "completions", + "got_raw": "completions", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "7", + "got": "7", + "got_raw": "7", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "CompleteStyle.COLUMN", + "got": "CompleteStyle.ESCAPE", + "got_raw": "CompleteStyle.ESCAPE", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.75 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "CompleteStyle.MULTI_COLUMN", + "got": "CompleteStyle.DYNAMIC", + "got_raw": "CompleteStyle.DYNAMIC", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6382978723404256 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "5", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "''", + "got": "Statement()", + "got_raw": "Statement()", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "1", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "3", + "got": "4", + "got_raw": "4", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "styled", + "got": "styled", + "got_raw": "styled", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "\"arg\")", + "got": "\"arg\"", + "got_raw": "\"arg\", \"arg\")", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "unstyled", + "got": "unstyled", + "got_raw": "unstyled", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "bar_text", + "got": "foo_text", + "got_raw": "foo_text", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.625 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "foo_text", + "got": "foo_text", + "got_raw": "foo_text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "\"tab: complete\")", + "got": "'bind \"Tab\" \"\\t\"')", + "got_raw": "'bind \"Tab\" \"\\t\"')", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.29411764705882354 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "original_completer)", + "got": "original_completer)", + "got_raw": "original_completer)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "\"bind ^I rl_complete\")", + "got": "\"bind -\"", + "got_raw": "\"bind -\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.4666666666666667 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "string", + "got": "string", + "got_raw": "string", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "input_string", + "got": "input_string", + "got_raw": "input_string", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "expected_line", + "got": "expected_line", + "got_raw": "expected_line", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "orig_cmd2_style", + "got": "orig_cmd2_style", + "got_raw": "orig_cmd2_style", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "orig_rich_style", + "got": "orig_rich_style", + "got_raw": "orig_rich_style", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "expected_output", + "got": "expected_output", + "got_raw": "expected_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "str(excinfo.value)", + "got": "str(excinfo)", + "got_raw": "str(excinfo)", + "exact_match": false, + "code_bleu": 0.4714045207910316, + "edit_similarity": 0.8 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "orig_argparse_style", + "got": "orig_argparse_style", + "got_raw": "orig_argparse_style", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "theme[cmd2_style_key]", + "got": "theme[cmd2_style_key]", + "got_raw": "theme[cmd2_style_key]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "theme[rich_style_key]", + "got": "theme[rich_style_key]", + "got_raw": "theme[rich_style_key]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "ru._from_ansi_wrapper.__func__", + "got": "Text.from_ansi", + "got_raw": "Text.from_ansi", + "exact_match": false, + "code_bleu": 0.2710806010829534, + "edit_similarity": 0.45454545454545453 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "out", + "got": "out", + "got_raw": "out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "err", + "got": "err", + "got_raw": "err", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "out2", + "got": "out2", + "got_raw": "out2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "err[0]", + "got": "err", + "got_raw": "err", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "err[1]", + "got": "err", + "got_raw": "err", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "'help'", + "got": "'help'", + "got_raw": "'help'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "out[0]", + "got": "out", + "got_raw": "out", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "s2", + "got": "s2", + "got_raw": "s2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "\"Red\"", + "got": "\"Red\"", + "got_raw": "\"Red\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "\"Bold\"", + "got": "\"Bold\"", + "got_raw": "\"Bold\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "my_str", + "got": "my_str", + "got_raw": "my_str", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "\"Orange\"", + "got": "\"Orange\"", + "got_raw": "\"Orange\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "your_str", + "got": "your_str", + "got_raw": "your_str", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "stripped", + "got": "stripped", + "got_raw": "stripped", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "micro_cf", + "got": "micro_cf", + "got_raw": "micro_cf", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "cursor_up", + "got": "cursor_up", + "got_raw": "cursor_up", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "cursor_pos", + "got": "cursor_pos", + "got_raw": "cursor_pos", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "hide_cursor", + "got": "hide_cursor", + "got_raw": "hide_cursor", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "show_cursor", + "got": "show_cursor", + "got_raw": "show_cursor", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "HELLO_WORLD", + "got": "HELLO_WORLD", + "got_raw": "HELLO_WORLD", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "text[:width]", + "got": "'---'", + "got_raw": "'---'", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "\"Custom RGB\"", + "got": "\"Custom RGB\"", + "got_raw": "\"Custom RGB\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "su.norm_fold(s2)", + "got": "s2", + "got_raw": "s2", + "exact_match": false, + "code_bleu": 0.09910059291689341, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "character * width", + "got": "character * width", + "got_raw": "character * width", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "\"Underline Green\"", + "got": "\"Underline Green\"", + "got_raw": "\"Underline Green\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "'\"' + my_str + '\"'", + "got": "'\"Hello World\"'", + "got_raw": "'\"Hello World\"'", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "3", + "got": "3.0", + "got_raw": "3.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "17", + "got": "17", + "got_raw": "17", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "''", + "got": "''", + "got_raw": "''", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "{}", + "got": "()", + "got_raw": "()", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "b''", + "got": "b''", + "got_raw": "b''", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "str", + "got": "str", + "got_raw": "str", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "'x'", + "got": "\"x\"", + "got_raw": "\"x\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "int", + "got": "int", + "got_raw": "int", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-cmd2/cmd2", + "expected": "ChildClass", + "got": "ChildClass", + "got_raw": "ChildClass", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "t[0]", + "got": "t.year", + "got_raw": "t.year", + "exact_match": false, + "code_bleu": 0.3388507513536918, + "edit_similarity": 0.2 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "t[1]", + "got": "t[1]", + "got_raw": "t[1]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "t[2]", + "got": "t[2]", + "got_raw": "t[2]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "t[3]", + "got": "t[3]", + "got_raw": "t[3]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "t[4]", + "got": "t[4]", + "got_raw": "t[4]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "d.day", + "got": "d.day", + "got_raw": "d.day", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "d.year", + "got": "d.year", + "got_raw": "d.year", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "d.hour", + "got": "d.hour", + "got_raw": "d.hour", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "p", + "got": "p", + "got_raw": "p", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "d1", + "got": "d1", + "got_raw": "d1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "d1", + "got": "d1", + "got_raw": "d1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "d2", + "got": "d2", + "got_raw": "d2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "d3", + "got": "d3", + "got_raw": "d3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "dt", + "got": "dt", + "got_raw": "dt", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "d1", + "got": "d1", + "got_raw": "d1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "d2", + "got": "d2", + "got_raw": "d2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "d3", + "got": "d2", + "got_raw": "d2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "tz", + "got": "tz", + "got_raw": "tz", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "t2", + "got": "t2", + "got_raw": "t2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "t3", + "got": "t3", + "got_raw": "t3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "dt2", + "got": "dt2", + "got_raw": "dt2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "it2", + "got": "it2", + "got_raw": "it2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "pi)", + "got": "pi", + "got_raw": "pi, timedelta(days=2 * 365.2425))", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.8 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "end", + "got": "end", + "got_raw": "end", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "1", + "got": "1.0", + "got_raw": "1.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "5", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "7", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "7", + "got": "-7", + "got_raw": "-7", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "7", + "got": "7", + "got_raw": "7", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "9", + "got": "9", + "got_raw": "9", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "6", + "got": "6", + "got_raw": "6", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "2", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "1", + "got": "delta.days", + "got_raw": "delta.days", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "6", + "got": "4", + "got_raw": "4", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "7", + "got": "-7", + "got_raw": "-7", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "7", + "got": "7", + "got_raw": "7", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "9", + "got": "9", + "got_raw": "9", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "6", + "got": "6", + "got_raw": "6", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "7", + "got": "7", + "got_raw": "7", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "8", + "got": "8", + "got_raw": "8", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "9", + "got": "9", + "got_raw": "9", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "0", + "got": "9", + "got_raw": "9", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "2", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "1", + "got": "11", + "got_raw": "11", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "8", + "got": "8", + "got_raw": "8", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "3", + "got": "23", + "got_raw": "23", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "1", + "got": "-1000001", + "got_raw": "-1000001", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "7", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "2", + "got": "-1", + "got_raw": "-1", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "4", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "5", + "got": "6", + "got_raw": "6", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "6", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "4", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "6", + "got": "6", + "got_raw": "6", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "0", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "3", + "got": "2.1", + "got_raw": "2.1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "1", + "got": "-2.0", + "got_raw": "-2.0", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "2", + "got": "-2", + "got_raw": "-2", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "1", + "got": "23", + "got_raw": "23", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "6", + "got": "6", + "got_raw": "6", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "7)", + "got": "7)", + "got_raw": "7)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "12", + "got": "12", + "got_raw": "12", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"1\"", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"3\"", + "got": "\"3\"", + "got_raw": "\"3\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"7\"", + "got": "\"7\"", + "got_raw": "\"7\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"0\"", + "got": "\"28\"", + "got_raw": "\"28\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"6\"", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"2\"", + "got": "\"2\"", + "got_raw": "\"2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"J\"", + "got": "\"J\"", + "got_raw": "\"J\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"3\"", + "got": "\"Q1\"", + "got_raw": "\"Q1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"8\"", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"3\"", + "got": "\"1.", + "got_raw": "\"1. rok\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"8\"", + "got": "\"3\"", + "got_raw": "\"3\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"fr\"", + "got": "\"fr\"", + "got_raw": "\"fr\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"25\"", + "got": "\"25\"", + "got_raw": "\"25\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"25\"", + "got": "\"25\"", + "got_raw": "\"25\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"po\"", + "got": "\"pon\"", + "got_raw": "\"pon\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"3.\"", + "got": "\"Qo\"", + "got_raw": "\"Qo\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"8.\"", + "got": "\"2.\"", + "got_raw": "\"2.\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"AM\"", + "got": "\"AM\"", + "got_raw": "\"AM\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"AM\"", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"1\u00b0\"", + "got": "\"1\u00b0\"", + "got_raw": "\"1\u00b0\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"\u6570\u79d2\"", + "got": "\"\u4eca\u304b\u3089", + "got_raw": "\"\u4eca\u304b\u3089 \u6570\u79d2\"", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.25 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"1\ucd08\"", + "got": "\"1\ucd08", + "got_raw": "\"1\ucd08 \ud6c4\"", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"2\ucd08\"", + "got": "\"1\ucd08\"", + "got_raw": "\"1\ucd08\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"3.\"", + "got": "\"q.", + "got_raw": "\"q. 1\"", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"8.\"", + "got": "\"1.\"", + "got_raw": "\"1.\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"zo\"", + "got": "\"zo\"", + "got_raw": "\"zo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"3.\"", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"8.\"", + "got": "\"3.\"", + "got_raw": "\"3.\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"AM\"", + "got": "\"AM\"", + "got_raw": "\"AM\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"29\"", + "got": "\"2\"", + "got_raw": "\"2\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"po\"", + "got": "\"pon\"", + "got_raw": "\"pon\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"AM\"", + "got": "\"AM\"", + "got_raw": "\"AM\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"29\"", + "got": "\"29\"", + "got_raw": "\"29\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"14\"", + "got": "\"14\"", + "got_raw": "\"14\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"15\"", + "got": "\"14:15:16\"", + "got_raw": "\"14:15:16\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"UTC\"", + "got": "\"UTC\"", + "got_raw": "\"UTC\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"UTC\"", + "got": "\"UTC\"", + "got_raw": "\"UTC\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"UTC\"", + "got": "\"UTC\"", + "got_raw": "\"UTC\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"UTC\"", + "got": "\"UTC\"", + "got_raw": "\"UTC\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"UTC\"", + "got": "\"UTC\"", + "got_raw": "\"UTC\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"srp\"", + "got": "\"srp\"", + "got_raw": "\"srp\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"29.\"", + "got": "\"2.\"", + "got_raw": "\"2.\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"28e\"", + "got": "\"28e\"", + "got_raw": "\"28e\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"dom\"", + "got": "\"dom.\"", + "got_raw": "\"dom.\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"ago\"", + "got": "\"ago\"", + "got_raw": "\"ago\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"28\u00b0\"", + "got": "\"28\"", + "got_raw": "\"28\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"28.\"", + "got": "\"28.\"", + "got_raw": "\"28.\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"28e\"", + "got": "\"28\"", + "got_raw": "\"28\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"29.\"", + "got": "\"29.\"", + "got_raw": "\"29.\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"sie\"", + "got": "\"sier.\"", + "got_raw": "\"sier.\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"1975\"", + "got": "\"1975\"", + "got_raw": "\"1975\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"1975\"", + "got": "\"1975\"", + "got_raw": "\"1975\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"dim.\"", + "got": "\"dim.\"", + "got_raw": "\"dim.\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"ao\u00fbt\"", + "got": "\"ao\u00fbt\"", + "got_raw": "\"ao\u00fbt\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"1 \u5206\u524d\"", + "got": "\"1 \u5206\u524d\"", + "got_raw": "\"1 \u5206\u524d\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"2 \u5206\u524d\"", + "got": "\"2 \u5206\u524d\"", + "got_raw": "\"2 \u5206\u524d\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"1 \u65e5\u524d\"", + "got": "\"1 \u65e5\u524d\"", + "got_raw": "\"1 \u65e5\u524d\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"1\ucd08 \uc804\"", + "got": "\"1\ucd08 \uc804\"", + "got_raw": "\"1\ucd08 \uc804\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"2\ucd08 \uc804\"", + "got": "\"2\ucd08 \uc804\"", + "got_raw": "\"2\ucd08 \uc804\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"1\ubd84 \uc804\"", + "got": "\"1\ubd84 \uc804\"", + "got_raw": "\"1\ubd84 \uc804\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"s\u00f8n.\"", + "got": "\"sun\"", + "got_raw": "\"sun\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"aug.\"", + "got": "\"aug.\"", + "got_raw": "\"aug.\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"aug.\"", + "got": "\"aug.\"", + "got_raw": "\"aug.\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"a.m.\"", + "got": "\"am\"", + "got_raw": "\"am\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"m\u00e5n.\"", + "got": "\"ma\"", + "got_raw": "\"ma\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"aug.\"", + "got": "\"aug.\"", + "got_raw": "\"aug.\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"test\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"+02:00\"", + "got": "\"Europe/Paris\"", + "got_raw": "\"Europe/Paris\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"+06:00\"", + "got": "\"UTC\"", + "got_raw": "\"UTC\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"+00:00\"", + "got": "\"UTC\"", + "got_raw": "\"UTC\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"1 week\"", + "got": "\"1 week\"", + "got_raw": "\"1 week\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"-1 day\"", + "got": "\"1 day ago\"", + "got_raw": "\"1 day ago\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"1 week\"", + "got": "\"1 week\"", + "got_raw": "\"1 week\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "\"-1 day\"", + "got": "\"1 day ago\"", + "got_raw": "\"1 day ago\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "hash(p)", + "got": "hash(d)", + "got_raw": "hash(d)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "p.dst()", + "got": "False", + "got_raw": "False", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "hash(d1)", + "got": "hash(d2)", + "got_raw": "hash(d2)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.875 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "hash(d3)", + "got": "hash(d3)", + "got_raw": "hash(d3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "dt.dst()", + "got": "False", + "got_raw": "False", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "hash(dt)", + "got": "hash(dt)", + "got_raw": "hash(dt)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "d.ctime()", + "got": "d.ctime()", + "got_raw": "d.ctime()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "dt.date()", + "got": "dt.date()", + "got_raw": "dt.date()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "dt.time()", + "got": "time(12", + "got_raw": "time(12, 34, 56, 123456)", + "exact_match": false, + "code_bleu": 0.3567621345008163, + "edit_similarity": 0.625 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "hash(dt2)", + "got": "hash(dt2)", + "got_raw": "hash(dt2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "hash(dt3)", + "got": "hash(dt3)", + "got_raw": "hash(dt3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "dt.date()", + "got": "dt.date()", + "got_raw": "dt.date()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-pendulum/pendulum", + "expected": "hash(dt1)", + "got": "hash(dt1)", + "got_raw": "hash(dt1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "str(ex.value)", + "got": "str(ex)", + "got_raw": "str(ex)", + "exact_match": false, + "code_bleu": 0.4714045207910316, + "edit_similarity": 0.7 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "luma.core.error.DeviceDisplayModeError)", + "got": "luma.core.error.DeviceDisplayModeError)", + "got_raw": "luma.core.error.DeviceDisplayModeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "7", + "got": "7", + "got_raw": "7", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "start_frame() + [0xE0, 0, 0, 0] * 7 + end_frame(7))", + "got": "[0] * 7)", + "got_raw": "[0] * 7)", + "exact_match": false, + "code_bleu": 0.20203050891044216, + "edit_similarity": 0.2711864406779661 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "start_frame() + [0xE6, 0, 0, 0xFF] * 6 + end_frame(6))", + "got": "", + "got_raw": "", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "start_frame() + [0xE4, 0x33, 0x22, 0x11] * 4 + end_frame(4))", + "got": "", + "got_raw": "", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "8", + "got": "8", + "got_raw": "8", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "32", + "got": "4", + "got_raw": "4", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "22)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[12, 0] * 5)", + "got": "[0x00] * 12)", + "got_raw": "[0x00] * 12)", + "exact_match": false, + "code_bleu": 0.25929058848145575, + "edit_similarity": 0.5833333333333334 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[12, 1] * 3)", + "got": "[0x00] * 12)", + "got_raw": "[0x00] * 12)", + "exact_match": false, + "code_bleu": 0.25929058848145575, + "edit_similarity": 0.5 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[10, 6] * 6)", + "got": "[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0", + "got_raw": "[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0", + "exact_match": false, + "code_bleu": 0.03026457500336624, + "edit_similarity": 0.08571428571428572 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "AssertionError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5384615384615384 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[ call([11, 7]), call([9, 0]), call([15, 0]), call([10, 7]), call([1, 0]), call([2, 0]), call([3, 0]), call([4, 0]), call([5, 0]), call([6, 0]), call([7, 0]), call([8, 0]), call([12, 1]) ])", + "got": "[call(0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00)])", + "got_raw": "[call(0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00)])", + "exact_match": false, + "code_bleu": 0.032818653904176556, + "edit_similarity": 0.26229508196721313 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[ call([1, 0x81, 1, 0xFF]), call([2, 0x81, 2, 0x81]), call([3, 0x81, 3, 0x81]), call([4, 0x81, 4, 0x81]), call([5, 0x81, 5, 0x81]), call([6, 0x81, 6, 0x81]), call([7, 0x81, 7, 0x81]), call([8, 0xFF, 8, 0x81]) ])", + "got": "[call(b\"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "got_raw": "[call(b\"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "exact_match": false, + "code_bleu": 0.02327171416418036, + "edit_similarity": 0.0058309037900874635 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[ call([1, 0x09, 1, 0x0F]), call([2, 0x09, 2, 0x09]), call([3, 0x09, 3, 0x09]), call([4, 0x09, 4, 0x09]), call([5, 0x09, 5, 0x09]), call([6, 0x09, 6, 0x09]), call([7, 0x09, 7, 0x09]), call([8, 0x0F, 8, 0x09]) ])", + "got": "[call(b\"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "got_raw": "[call(b\"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "exact_match": false, + "code_bleu": 0.02327171416418036, + "edit_similarity": 0.0058309037900874635 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[ call([1, 0x00, 1, 0x00]), call([2, 0x00, 2, 0x00]), call([3, 0x00, 3, 0x00]), call([4, 0x00, 4, 0x00]), call([5, 0xFF, 5, 0xFF]), call([6, 0x80, 6, 0x01]), call([7, 0x80, 7, 0x01]), call([8, 0xFF, 8, 0xFF]) ])", + "got": "[call(b\"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "got_raw": "[call(b\"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "exact_match": false, + "code_bleu": 0.02327171416418036, + "edit_similarity": 0.0058309037900874635 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[ call([1, 0xFF, 1, 0xFF]), call([2, 0x01, 2, 0x80]), call([3, 0x01, 3, 0x80]), call([4, 0xFF, 4, 0xFF]), call([5, 0x00, 5, 0x00]), call([6, 0x00, 6, 0x00]), call([7, 0x00, 7, 0x00]), call([8, 0x00, 8, 0x00]) ])", + "got": "[call(b\"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "got_raw": "[call(b\"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "exact_match": false, + "code_bleu": 0.02327171416418036, + "edit_similarity": 0.0058309037900874635 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[ call([1, 0xF0, 1, 0x90]), call([2, 0x90, 2, 0x90]), call([3, 0x90, 3, 0x90]), call([4, 0x90, 4, 0x90]), call([5, 0x90, 5, 0x90]), call([6, 0x90, 6, 0x90]), call([7, 0x90, 7, 0x90]), call([8, 0x90, 8, 0xF0]) ])", + "got": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x", + "got_raw": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x", + "exact_match": false, + "code_bleu": 0.021000437459062837, + "edit_similarity": 0.005194805194805195 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[ call([1, 15, 1, 9, 1, 0, 1, 0]), call([2, 9, 2, 9, 2, 0, 2, 0]), call([3, 9, 3, 9, 3, 0, 3, 0]), call([4, 9, 4, 9, 4, 0, 4, 0]), call([5, 9, 5, 9, 5, 0, 5, 0]), call([6, 9, 6, 9, 6, 0, 6, 0]), call([7, 9, 7, 9, 7, 0, 7, 0]), call([8, 9, 8, 15, 8, 0, 8, 0]) ])", + "got": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x", + "got_raw": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x", + "exact_match": false, + "code_bleu": 0.013094390415650945, + "edit_similarity": 0.004597701149425287 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[ call([11, 7, 11, 7]), call([9, 0, 9, 0]), call([15, 0, 15, 0]), call([10, 7, 10, 7]), call([1, 0, 1, 0]), call([2, 0, 2, 0]), call([3, 0, 3, 0]), call([4, 0, 4, 0]), call([5, 0, 5, 0]), call([6, 0, 6, 0]), call([7, 0, 7, 0]), call([8, 0, 8, 0]), call([12, 1, 12, 1]) ])", + "got": "[call(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00)]", + "got_raw": "[call(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00)]", + "exact_match": false, + "code_bleu": 0.021360967253812608, + "edit_similarity": 0.005361930294906166 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[ call([1, 0x80, 1, 0xFF, 1, 0x01, 1, 0xFF]), call([2, 0x80, 2, 0x80, 2, 0x01, 2, 0x01]), call([3, 0x80, 3, 0x80, 3, 0x01, 3, 0x01]), call([4, 0x80, 4, 0x80, 4, 0x01, 4, 0x01]), call([5, 0x80, 5, 0x80, 5, 0x01, 5, 0x01]), call([6, 0x80, 6, 0x80, 6, 0x01, 6, 0x01]), call([7, 0x80, 7, 0x80, 7, 0x01, 7, 0x01]), call([8, 0xFF, 8, 0x80, 8, 0xFF, 8, 0x01]) ])", + "got": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x", + "got_raw": "[call(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x", + "exact_match": false, + "code_bleu": 0.013094390415650945, + "edit_similarity": 0.003780718336483932 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "str(ex.value)", + "got": "str(ex)", + "got_raw": "str(ex)", + "exact_match": false, + "code_bleu": 0.4714045207910316, + "edit_similarity": 0.7 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "OverflowError)", + "got": "luma.core.error.OverflowError)", + "got_raw": "luma.core.error.OverflowError)", + "exact_match": false, + "code_bleu": 0.21105340631872638, + "edit_similarity": 0.6363636363636364 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "neoseg.device.image)", + "got": "neoseg.image)", + "got_raw": "neoseg.image)", + "exact_match": false, + "code_bleu": 0.5065571237677283, + "edit_similarity": 0.7878787878787878 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "luma.core.error.DeviceDisplayModeError)", + "got": "luma.core.error.DimensionError)", + "got_raw": "luma.core.error.DimensionError)", + "exact_match": false, + "code_bleu": 0.7506238537503395, + "edit_similarity": 0.7714285714285715 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[0x7f, 0x7f]", + "got": "[(\"B\", \"B\")]", + "got_raw": "[(\"B\", \"B\")]", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[0x7f, 0x08, 0x7f]", + "got": "[(\"B\", \"B\"), (\"B\", \"_\")]", + "got_raw": "[(\"B\", \"B\"), (\"B\", \"_\")]", + "exact_match": false, + "code_bleu": 0.12011055432195765, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[0x80, 0x67, 0x7e, 0x47]", + "got": "[(\"D\", \"ot\", \"P\", \"F\")]", + "got_raw": "[(\"D\", \"ot\", \"P\", \"F\")]", + "exact_match": false, + "code_bleu": 0.14991106946711685, + "edit_similarity": 0.3404255319148936 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[0x76, 0x1d | 0x80, 0x80, 0x80]", + "got": "[(\"N\", \"o...\"), (\"...\", \"...\")]", + "got_raw": "[(\"N\", \"o...\"), (\"...\", \"...\")]", + "exact_match": false, + "code_bleu": 0.12571192676522522, + "edit_similarity": 0.25806451612903225 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[0x79, 0x80, 0x30, 0x33, 0x30, 0x5b, 0x7b]", + "got": "[(\"3\", \"1\", \"4\", \"1\", \"5\", \"9\")]", + "got_raw": "[(\"3\", \"1\", \"4\", \"1\", \"5\", \"9\")]", + "exact_match": false, + "code_bleu": 0.11633270842295028, + "edit_similarity": 0.32432432432432434 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[0x79 | 0x80, 0x30, 0x33, 0x30, 0x5b, 0x7b]", + "got": "[(\"3\", \"1\", \"4\", \"1\", \"5\", \"9\")]", + "got_raw": "[(\"3\", \"1\", \"4\", \"1\", \"5\", \"9\")]", + "exact_match": false, + "code_bleu": 0.11633270842295028, + "edit_similarity": 0.3466666666666667 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[0x6d, 0x7b | 0x80, 0x30, 0x6d, 0x63, 0x4e]", + "got": "[ (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,", + "got_raw": "[ (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ", + "exact_match": false, + "code_bleu": 0.01902591407689958, + "edit_similarity": 0.1871345029239766 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[0x30, 0x6d, 0x70, 0x80, 0x7e, 0x80, 0x7e, 0x80, 0x30]", + "got": "[(\"1\", \"27.0.0.1\"), (\"127.0.0.1\", \"1\")]", + "got_raw": "[(\"1\", \"27.0.0.1\"), (\"127.0.0.1\", \"1\")]", + "exact_match": false, + "code_bleu": 0.08601342357620673, + "edit_similarity": 0.23655913978494625 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[0x00, 0x00, 0x5b, 0x6d, 0x5b, 0x7b, 0x6d, 0x7e | 0x80]", + "got": "[(\" 525920\", \"0\")]", + "got_raw": "[(\" 525920\", \"0\")]", + "exact_match": false, + "code_bleu": 0.08138270417847641, + "edit_similarity": 0.16216216216216217 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[0x30, 0x6d, 0x70 | 0x80, 0x7e | 0x80, 0x7e | 0x80, 0x30]", + "got": "[(\"1\", \"27.0.0.1\"), (\"127.0.0.1\", \"1\")]", + "got_raw": "[(\"1\", \"27.0.0.1\"), (\"127.0.0.1\", \"1\")]", + "exact_match": false, + "code_bleu": 0.08601342357620673, + "edit_similarity": 0.25 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[0x37, 0x6f, 0x06, 0x06, 0x1d, 0x00, 0x14, 0x1d, 0x05, 0x06, 0x3d]", + "got": "[(\"H\", \"e\", \"l\", \"o\", \" \", \"w\", \"r\", \"l\", \"d\")]", + "got_raw": "[(\"H\", \"e\", \"l\", \"o\", \" \", \"w\", \"r\", \"l\", \"d\")]", + "exact_match": false, + "code_bleu": 0.08123313927220152, + "edit_similarity": 0.336283185840708 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "16", + "got": "16", + "got_raw": "16", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[0x72] + [0] * 256 * 3)", + "got": "0)", + "got_raw": "0)", + "exact_match": false, + "code_bleu": 0.09910059291689341, + "edit_similarity": 0.16 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[0x72] + [0xFF] * 256 * 3)", + "got": "b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\")", + "got_raw": "b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\")", + "exact_match": false, + "code_bleu": 0.1129502504512306, + "edit_similarity": 0.12903225806451613 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[0x72] + [0x6B] * 256 * 3)", + "got": "b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\")", + "got_raw": "b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\")", + "exact_match": false, + "code_bleu": 0.1129502504512306, + "edit_similarity": 0.12903225806451613 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[0x72] + get_json_data('demo_unicornhathd'))", + "got": "device.data)", + "got_raw": "device.data)", + "exact_match": false, + "code_bleu": 0.15969071642803745, + "edit_similarity": 0.21428571428571427 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[0x72] + get_json_data('demo_unicornhathd_alphablend'))", + "got": "device._serial_data)", + "got_raw": "device._serial_data)", + "exact_match": false, + "code_bleu": 0.15969071642803745, + "edit_similarity": 0.24 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "7", + "got": "7", + "got_raw": "7", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "32", + "got": "False", + "got_raw": "False", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "0x6B)", + "got": "0x6B)", + "got_raw": "0x6B)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "str(ex.value)", + "got": "str(ex)", + "got_raw": "str(ex)", + "exact_match": false, + "code_bleu": 0.4714045207910316, + "edit_similarity": 0.7 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "RuntimeError)", + "got": "RuntimeError)", + "got_raw": "RuntimeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[ call(chan, i, 0x000000) for i in range(7)])", + "got": "[call(leds, 0) for _ in range(7)])", + "got_raw": "[call(leds, 0) for _ in range(7)])", + "exact_match": false, + "code_bleu": 0.48106022256052067, + "edit_similarity": 0.7341772151898734 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[ call(chan, i, 0x000000) for i in range(32)])", + "got": "[call(leds, 0, 0)])", + "got_raw": "[call(leds, 0, 0)])", + "exact_match": false, + "code_bleu": 0.19952600730504763, + "edit_similarity": 0.4 + }, + { + "repo": "rm-hull/luma.led_matrix", + "expected": "[ call(chan, 0, 0xFF0000), call(chan, 1, 0xFF0000), call(chan, 2, 0xFF0000), call(chan, 3, 0xFF0000), call(chan, 4, 0xFF0000), call(chan, 5, 0x000000), call(chan, 6, 0x000000), call(chan, 7, 0xFF0000), call(chan, 8, 0xFF0000), call(chan, 9, 0x000000), call(chan, 10, 0x000000), call(chan, 11, 0xFF0000), call(chan, 12, 0xFF0000), call(chan, 13, 0xFF0000), call(chan, 14, 0xFF0000), call(chan, 15, 0xFF0000), ])", + "got": "[call(leds, 0, 0, 0, 0, 0, 0)])", + "got_raw": "[call(leds, 0, 0, 0, 0, 0, 0)])", + "exact_match": false, + "code_bleu": 0.027887536109339642, + "edit_similarity": 0.013605442176870748 + }, + { + "repo": "siboehm/lleaves", + "expected": "np.float32", + "got": "np.float32", + "got_raw": "np.float32", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "llvm_pred)", + "got": "llvm_pred)", + "got_raw": "llvm_pred)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "llvm.predict(data, n_jobs=2))", + "got": "lgb.predict(data), llvm.predict(data))", + "got_raw": "lgb.predict(data), llvm.predict(data))", + "exact_match": false, + "code_bleu": 0.375022891676693, + "edit_similarity": 0.5671641791044776 + }, + { + "repo": "siboehm/lleaves", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "[results[res_idx]]", + "got": "res_idx", + "got_raw": "res_idx", + "exact_match": false, + "code_bleu": 0.09910059291689341, + "edit_similarity": 0.56 + }, + { + "repo": "siboehm/lleaves", + "expected": "list(df2[\"C1\"].cat.codes)", + "got": "list(df2[\"C1\"].cat.codes)", + "got_raw": "list(df2[\"C1\"].cat.codes)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "list(df2[\"C2\"].cat.codes)", + "got": "list(df2[\"C2\"].cat.codes)", + "got_raw": "list(df2[\"C2\"].cat.codes)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "lgbm_model.predict(data))", + "got": "lightgbm_model.predict(data))", + "got_raw": "lightgbm_model.predict(data))", + "exact_match": false, + "code_bleu": 0.8408964152537145, + "edit_similarity": 0.9259259259259259 + }, + { + "repo": "siboehm/lleaves", + "expected": "lgbm_model.predict([input])", + "got": "lgbm_model.predict([input])", + "got_raw": "lgbm_model.predict([input])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "Path(model_path).read_text()", + "got": "model_path.read_text()", + "got_raw": "model_path.read_text()", + "exact_match": false, + "code_bleu": 0.49704420547940664, + "edit_similarity": 0.88 + }, + { + "repo": "siboehm/lleaves", + "expected": "lgbm_model.predict(tests_data))", + "got": "llvm_model.predict(tests_data)", + "got_raw": "llvm_model.predict(tests_data), lgbm_model.predict(tests_data))", + "exact_match": false, + "code_bleu": 0.693520038488761, + "edit_similarity": 0.9180327868852459 + }, + { + "repo": "siboehm/lleaves", + "expected": "stdout", + "got": "stdout", + "got_raw": "stdout", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "lgbm_model.predict(data, n_jobs=2))", + "got": "ensordot(data, np.array([1.0] * 3, dtype=np.float64))", + "got_raw": "ensordot(data, np.array([1.0] * 3, dtype=np.float64))", + "exact_match": false, + "code_bleu": 0.13817790393734294, + "edit_similarity": 0.3409090909090909 + }, + { + "repo": "siboehm/lleaves", + "expected": "lgbm_model.predict(NYC_data[:1000], n_jobs=2))", + "got": "ensordot(lgbm_model.predict(NYC_data), np.ones_like(lgbm_model.predict(NYC_data)))", + "got_raw": "ensordot(lgbm_model.predict(NYC_data), np.ones_like(lgbm_model.predict(NYC_data)))", + "exact_match": false, + "code_bleu": 0.24648321974767612, + "edit_similarity": 0.53125 + }, + { + "repo": "siboehm/lleaves", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "5", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "[]", + "got": "[[\"a\"", + "got_raw": "[[\"a\", \"b\", \"c\"], [\"b\", \"c\", \"d\"], [\"w\", \"x\", \"y\", \"z\"]]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "siboehm/lleaves", + "expected": "orig)", + "got": "orig)", + "got_raw": "orig)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "orig.dtype", + "got": "orig.dtype", + "got_raw": "orig.dtype", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "lgbm_model.predict(df))", + "got": "a", + "got_raw": "a", + "exact_match": false, + "code_bleu": 0.07142857142857142, + "edit_similarity": 0.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "lgbm_model.predict(sliced))", + "got": "sliced)", + "got_raw": "sliced)", + "exact_match": false, + "code_bleu": 0.20203050891044216, + "edit_similarity": 0.4117647058823529 + }, + { + "repo": "siboehm/lleaves", + "expected": "[ [\"a\", \"b\", \"c\"], [\"b\", \"c\", \"d\"], [\"w\", \"x\", \"y\", \"z\"], ]", + "got": "[[\"a\", \"b\", \"c\"], [\"b\", \"c\", \"d\"], [\"w\", \"x\", \"y\", \"z\"]]", + "got_raw": "[[\"a\", \"b\", \"c\"], [\"b\", \"c\", \"d\"], [\"w\", \"x\", \"y\", \"z\"]]", + "exact_match": false, + "code_bleu": 0.9363055597212666, + "edit_similarity": 0.9739130434782609 + }, + { + "repo": "siboehm/lleaves", + "expected": "res)", + "got": "tiniest_llvm.predict([3", + "got_raw": "tiniest_llvm.predict([3 * [0.0], 3 * [1.0], 3 * [-1.0]]))", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.14814814814814814 + }, + { + "repo": "siboehm/lleaves", + "expected": "ValueError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "siboehm/lleaves", + "expected": "lgbm.predict(arr)", + "got": "lgbm.predict(arr)", + "got_raw": "lgbm.predict(arr)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "err1.value.args[0]", + "got": "err1.value.args[0]", + "got_raw": "err1.value.args[0]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "err2.value.args[0]", + "got": "err2.value.args[0]", + "got_raw": "err2.value.args[0]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "pure_cat_llvm.predict([3 * [0.0], 3 * [1.0], 3 * [-1.0]]))", + "got": "ensordot(res, np.array([1.0, 1.0, 1.0]))", + "got_raw": "ensordot(res, np.array([1.0, 1.0, 1.0]))", + "exact_match": false, + "code_bleu": 0.13462815347453236, + "edit_similarity": 0.40816326530612246 + }, + { + "repo": "siboehm/lleaves", + "expected": "lgbm_model.predict(data))", + "got": "lgbm_model.predict(data))", + "got_raw": "lgbm_model.predict(data))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "llvm_model.predict(data))", + "got": "llvm_model.predict(data))", + "got_raw": "llvm_model.predict(data))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "llvm_model.predict(X))", + "got": "bst.predict(X))", + "got_raw": "bst.predict(X))", + "exact_match": false, + "code_bleu": 0.8408964152537145, + "edit_similarity": 0.6486486486486487 + }, + { + "repo": "siboehm/lleaves", + "expected": "llvm_model.predict(data))", + "got": "llvm_model.predict(data))", + "got_raw": "llvm_model.predict(data))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "NYC_lgbm.predict(data))", + "got": "NYC_lgbm.predict(data))", + "got_raw": "NYC_lgbm.predict(data))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "4", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "12", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "siboehm/lleaves", + "expected": "18", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "siboehm/lleaves", + "expected": "95", + "got": "[", + "got_raw": "[ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "np.float64", + "got": "lgbm_result.dtype", + "got_raw": "lgbm_result.dtype", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.14814814814814814 + }, + { + "repo": "siboehm/lleaves", + "expected": "np.float32", + "got": "np.float32", + "got_raw": "np.float32", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "llvm_result)", + "got": "llvm_result)", + "got_raw": "llvm_result)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "lightgbm_model.num_trees()", + "got": "lightgbm_model.num_trees()", + "got_raw": "lightgbm_model.num_trees()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "llvm.predict(X, n_jobs=2))", + "got": "llvm.predict(X), lgbm.predict(X))", + "got_raw": "llvm.predict(X), lgbm.predict(X))", + "exact_match": false, + "code_bleu": 0.375022891676693, + "edit_similarity": 0.6440677966101694 + }, + { + "repo": "siboehm/lleaves", + "expected": "lightgbm_model.num_feature()", + "got": "lightgbm_model.num_feature()", + "got_raw": "lightgbm_model.num_feature()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "llvm.num_model_per_iteration()", + "got": "llvm.num_model_per_iteration()", + "got_raw": "llvm.num_model_per_iteration()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "siboehm/lleaves", + "expected": "lgbm_model.predict([input_data]))", + "got": "llvm_model.predict(input_data))", + "got_raw": "llvm_model.predict(input_data))", + "exact_match": false, + "code_bleu": 0.33233433830818204, + "edit_similarity": 0.90625 + }, + { + "repo": "siboehm/lleaves", + "expected": "lightgbm_model.predict(input_data))", + "got": "(lightgbm_model.predict(input_data))", + "got_raw": "(lightgbm_model.predict(input_data))", + "exact_match": false, + "code_bleu": 0.8633400213704505, + "edit_similarity": 0.9859154929577465 + }, + { + "repo": "siboehm/lleaves", + "expected": "lightgbm_model.num_model_per_iteration()", + "got": "llvm_model.num_feature()", + "got_raw": "llvm_model.num_feature()", + "exact_match": false, + "code_bleu": 0.3860973950960897, + "edit_similarity": 0.5625 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "test_session", + "got": "test_session", + "got_raw": "test_session", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "YTMusicUserError, match=\"Invalid auth\")", + "got": "YTMusicUserError)", + "got_raw": "YTMusicUserError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.6071428571428571 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "0", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "second_json", + "got": "second_json", + "got_raw": "second_json", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "second_token", + "got": "second_token", + "got_raw": "second_token", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "first_expire", + "got": "first_expire", + "got_raw": "first_expire", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "time.time() + 60", + "got": "first_expire", + "got_raw": "first_expire", + "exact_match": false, + "code_bleu": 0.07142857142857142, + "edit_similarity": 0.21428571428571427 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "YTMusicUserError)", + "got": "YTMusicUserError)", + "got_raw": "YTMusicUserError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "AuthType.OAUTH_CUSTOM_FULL", + "got": "AuthType.OAUTH_CUSTOM", + "got_raw": "AuthType.OAUTH_CUSTOM", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.8936170212765957 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "AuthType.OAUTH_CUSTOM_CLIENT", + "got": "AuthType.OAUTH", + "got_raw": "AuthType.OAUTH", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "3", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "9", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "7", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "5", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "3", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "4", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "5", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "25", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "50", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "40", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "5", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "21", + "got": "21", + "got_raw": "21", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "10", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "34", + "got": "22", + "got_raw": "22", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "46", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "456", + "got": "500", + "got_raw": "500", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "100", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "255", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "False", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "playlist", + "got": "playlist", + "got_raw": "playlist", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "tracks_len", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "5", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "9", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "10", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "20", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "80", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "150", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "100", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "5", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "3", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "4", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "20", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "25", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "40", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "409", + "got": "200", + "got_raw": "200", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "ResponseStatus.SUCCEEDED", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.30120066786994826, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "config.getint(\"limits\", \"library_upload_albums\")", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.0625, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "config.getint(\"limits\", \"library_upload_artists\")", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.0625, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "Exception, match=\"The provided file does not exist\")", + "got": "YTMusicUserError)", + "got_raw": "YTMusicUserError)", + "exact_match": false, + "code_bleu": 0.17909498863725698, + "edit_similarity": 0.14492753623188406 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "YTMusicUserError, match=\"larger than the limit of 300MB\")", + "got": "YTMusicUserError)", + "got_raw": "YTMusicUserError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.4594594594594595 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "Exception, match=\"The provided file type is not supported\")", + "got": "YTMusicUserError)", + "got_raw": "YTMusicUserError)", + "exact_match": false, + "code_bleu": 0.17909498863725698, + "edit_similarity": 0.15789473684210525 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "90", + "got": "90", + "got_raw": "90", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "45", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "25", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "config.getint(\"playlists\", \"own_length\")", + "got": "config.getint(\"playlists\", \"own_track_length\")", + "got_raw": "config.getint(\"playlists\", \"own_track_length\")", + "exact_match": false, + "code_bleu": 0.7506238537503395, + "edit_similarity": 0.9302325581395349 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "config.getint(\"albums\", \"album_track_length\")", + "got": "25", + "got_raw": "25", + "exact_match": false, + "code_bleu": 0.0625, + "edit_similarity": 0.0 + }, + { + "repo": "sigma67/ytmusicapi", + "expected": "Exception, match=\"No content returned by the server\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.17909498863725698, + "edit_similarity": 0.1875 + }, + { + "repo": "supabase/supabase-py", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "5", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "res.data", + "got": "res", + "got_raw": "res", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "supabase/supabase-py", + "expected": "res.data", + "got": "res", + "got_raw": "res", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "supabase/supabase-py", + "expected": "user.email", + "got": "user.email", + "got_raw": "user.email", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "user.email", + "got": "user.email", + "got_raw": "user.email", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "err[\"code\"]", + "got": "\"unexpected_failure\"", + "got_raw": "\"unexpected_failure\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.3225806451612903 + }, + { + "repo": "supabase/supabase-py", + "expected": "dict[str, str]", + "got": "Union[None, Dict[str, str]]", + "got_raw": "Union[None, Dict[str, str]]", + "exact_match": false, + "code_bleu": 0.41722614486115056, + "edit_similarity": 0.6341463414634146 + }, + { + "repo": "supabase/supabase-py", + "expected": "client.headers", + "got": "client.headers", + "got_raw": "client.headers", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "client.headers", + "got": "client.headers", + "got_raw": "client.headers", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "credentials.email", + "got": "credentials.email", + "got_raw": "credentials.email", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "credentials.email", + "got": "credentials.email", + "got_raw": "credentials.email", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "{\"test\": \"data\"}", + "got": "{\"test\": \"data\"}", + "got_raw": "{\"test\": \"data\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "{\"test\": \"data\"}", + "got": "{\"test\": \"data\"}", + "got_raw": "{\"test\": \"data\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "{\"name\": \"Michael\"}", + "got": "{\"name\": \"John Doe\"}", + "got_raw": "{\"name\": \"John Doe\"}", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.717948717948718 + }, + { + "repo": "supabase/supabase-py", + "expected": "{\"name\": \"Michael\"}", + "got": "{\"name\": \"John Doe\"}", + "got_raw": "{\"name\": \"John Doe\"}", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.717948717948718 + }, + { + "repo": "supabase/supabase-py", + "expected": "{\"message\": \"success\"}", + "got": "{\"message\": \"success\"}", + "got_raw": "{\"message\": \"success\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "{\"message\": \"success\"}", + "got": "{\"message\": \"success\"}", + "got_raw": "{\"message\": \"success\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "{\"id\": 2, \"name\": \"Jane\"}", + "got": "{\"id\": 2, \"name\": \"John\"}", + "got_raw": "{\"id\": 2, \"name\": \"John\"}", + "exact_match": false, + "code_bleu": 0.7825422900366437, + "edit_similarity": 0.92 + }, + { + "repo": "supabase/supabase-py", + "expected": "{\"id\": 2, \"name\": \"Jane\"}", + "got": "{\"id\": 2, \"name\": \"John\"}", + "got_raw": "{\"id\": 2, \"name\": \"John\"}", + "exact_match": false, + "code_bleu": 0.7825422900366437, + "edit_similarity": 0.92 + }, + { + "repo": "supabase/supabase-py", + "expected": "{\"id\": 1, \"name\": \"Michael\"}", + "got": "{\"id\": \"1\", \"name\": \"John\"}", + "got_raw": "{\"id\": \"1\", \"name\": \"John\"}", + "exact_match": false, + "code_bleu": 0.392814650900513, + "edit_similarity": 0.8 + }, + { + "repo": "supabase/supabase-py", + "expected": "{\"id\": 1, \"name\": \"Michael\"}", + "got": "{\"id\": \"1\"}", + "got_raw": "{\"id\": \"1\"}", + "exact_match": false, + "code_bleu": 0.2984916477287617, + "edit_similarity": 0.5128205128205128 + }, + { + "repo": "supabase/supabase-py", + "expected": "[{\"id\": 2, \"room_name\": \"Topaz\"}]", + "got": "[", + "got_raw": "[", + "exact_match": false, + "code_bleu": 0.05405486886376005, + "edit_similarity": 0.058823529411764705 + }, + { + "repo": "supabase/supabase-py", + "expected": "[{\"id\": 2, \"room_name\": \"Topaz\"}]", + "got": "[{\"id\": \"1\", \"room_name\": \"A\"}]", + "got_raw": "[{\"id\": \"1\", \"room_name\": \"A\"}]", + "exact_match": false, + "code_bleu": 0.46595384151899627, + "edit_similarity": 0.84375 + }, + { + "repo": "supabase/supabase-py", + "expected": "[{\"id\": 1, \"room_name\": \"Emerald\"}]", + "got": "[ {\"id\": \"1\", \"room_name\": \"A\"} ]", + "got_raw": "[ {\"id\": \"1\", \"room_name\": \"A\"} ]", + "exact_match": false, + "code_bleu": 0.46595384151899627, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "supabase/supabase-py", + "expected": "str", + "got": "str", + "got_raw": "str", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "res", + "got": "\"EXPLAIN", + "got_raw": "\"EXPLAIN (ANALYZE, VERBOSE, SETS, BUFFERS, WAL)\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "res", + "got": "\"EXPLAIN", + "got_raw": "\"EXPLAIN (ANALYZE, VERBOSE, SETS, BUFFERS, WAL)\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "key", + "got": "key", + "got_raw": "key", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "key", + "got": "key", + "got_raw": "key", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "bool", + "got": "bool", + "got_raw": "bool", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "code", + "got": "code", + "got_raw": "code", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "path", + "got": "path", + "got_raw": "path", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "path", + "got": "path", + "got_raw": "path", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "email", + "got": "email", + "got_raw": "email", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "email", + "got": "email", + "got_raw": "email", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "path)", + "got": "request.assert_called_once_with(method", + "got_raw": " request.assert_called_once_with(method, path)", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.13953488372093023 + }, + { + "repo": "supabase/supabase-py", + "expected": "path)", + "got": "request.assert_called_once_with(method", + "got_raw": " request.assert_called_once_with(method, path)", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.13953488372093023 + }, + { + "repo": "supabase/supabase-py", + "expected": "emails", + "got": "emails", + "got_raw": "emails", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "emails", + "got": "emails", + "got_raw": "emails", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "status", + "got": "status", + "got_raw": "status", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "copied", + "got": "copied", + "got_raw": "copied", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "params", + "got": "URL(expected_url).query", + "got_raw": "URL(expected_url).query", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.13793103448275862 + }, + { + "repo": "supabase/supabase-py", + "expected": "copied", + "got": "copied", + "got_raw": "copied", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "params", + "got": "URL(expected_url).query", + "got_raw": "URL(expected_url).query", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.13793103448275862 + }, + { + "repo": "supabase/supabase-py", + "expected": "message", + "got": "message", + "got_raw": "message", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "message", + "got": "message", + "got_raw": "message", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "callback", + "got": "callback", + "got_raw": "callback", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "client_id", + "got": "client_id", + "got_raw": "client_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "client_id", + "got": "client_id", + "got_raw": "client_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "valid_url", + "got": "valid_url", + "got_raw": "valid_url", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "valid_url", + "got": "valid_url", + "got_raw": "valid_url", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "valid_url", + "got": "valid_url", + "got_raw": "valid_url", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "APIError)", + "got": "APIError)", + "got_raw": "APIError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "APIError)", + "got": "APIError)", + "got_raw": "APIError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "bucket_id", + "got": "bucket_id", + "got_raw": "bucket_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "bucket_id", + "got": "bucket_id", + "got_raw": "bucket_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "actual_url", + "got": "str(actual_url)", + "got_raw": "str(actual_url)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.8 + }, + { + "repo": "supabase/supabase-py", + "expected": "actual_url", + "got": "str(actual_url)", + "got_raw": "str(actual_url)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.8 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"*\"", + "got": "\"count\"", + "got_raw": "\"count\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"2\"", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"0\"", + "got": "\"0\"", + "got_raw": "\"0\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"1\"", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"*\"", + "got": "\"count\"", + "got_raw": "\"count\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"2\"", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"0\"", + "got": "\"0\"", + "got_raw": "\"0\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"1\"", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"any\"", + "got": "\"any\"", + "got_raw": "\"any\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"GET\"", + "got": "\"GET\"", + "got_raw": "\"GET\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"GET\"", + "got": "\"GET\"", + "got_raw": "\"GET\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"GET\"", + "got": "\"GET\"", + "got_raw": "\"GET\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"GET\"", + "got": "\"GET\"", + "got_raw": "\"GET\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"1.0\"", + "got": "\"1.0\"", + "got_raw": "\"1.0\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"1.0\"", + "got": "\"1.0\"", + "got_raw": "\"1.0\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"apple\"", + "got": "\"apple\"", + "got_raw": "\"apple\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"apple\"", + "got": "\"apple\"", + "got_raw": "\"apple\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"eq.val\"", + "got": "\"eq\"", + "got_raw": "\"eq\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"x=eq.a\"", + "got": "\"?\"", + "got_raw": "\"?\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"x=gt.a\"", + "got": "\"{'x_gt':", + "got_raw": "\"{'x_gt': 'a'}\"", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"x=cs.a\"", + "got": "\"?\"", + "got_raw": "\"?\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"x=is.a\"", + "got": "\"{'x':", + "got_raw": "\"{'x': 'a'}\"", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"eq.val\"", + "got": "\"eq\"", + "got_raw": "\"eq\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"x=eq.a\"", + "got": "\"=x=a\"", + "got_raw": "\"=x=a\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"x=gt.a\"", + "got": "\"{'x_gt':", + "got_raw": "\"{'x_gt': 'a'}\"", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"x=cs.a\"", + "got": "\"{'x':", + "got_raw": "\"{'x': 'a'}\"", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"x=is.a\"", + "got": "\"{'x':", + "got_raw": "\"{'x': 'a'}\"", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"grapes\"", + "got": "\"grapes\"", + "got_raw": "\"grapes\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"grapes\"", + "got": "\"apple\"", + "got_raw": "\"apple\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"x=neq.a\"", + "got": "\"?\"", + "got_raw": "\"?\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"x=gte.a\"", + "got": "\"{'x_gte':", + "got_raw": "\"{'x_gte': 'a'}\"", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.5263157894736842 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"x=neq.a\"", + "got": "\"?\"", + "got_raw": "\"?\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"x=gte.a\"", + "got": "\"{'x_gte':", + "got_raw": "\"{'x_gte': 'a'}\"", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.5263157894736842 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"user123\"", + "got": "\"user123\"", + "got_raw": "\"user123\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"bucket1\"", + "got": "\"bucket1\"", + "got_raw": "\"bucket1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"bucket2\"", + "got": "\"bucket2\"", + "got_raw": "\"bucket2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"bucket\")", + "got": "\"\")", + "got_raw": "\"\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"bucket1\"", + "got": "\"bucket1\"", + "got_raw": "\"bucket1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"bucket2\"", + "got": "\"bucket2\"", + "got_raw": "\"bucket2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"bucket\")", + "got": "\"\")", + "got_raw": "\"\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"eu-west-1\"", + "got": "\"eu-west-1\"", + "got_raw": "\"eu-west-1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"eu-west-2\"", + "got": "\"eu-west-2\"", + "got_raw": "\"eu-west-2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"eu-west-3\"", + "got": "\"eu-west-3\"", + "got_raw": "\"eu-west-3\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"sa-east-1\"", + "got": "\"sa-east-1\"", + "got_raw": "\"sa-east-1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"us-east-1\"", + "got": "\"us-east-1\"", + "got_raw": "\"us-east-1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"us-east-1\"", + "got": "\"us-east-1\"", + "got_raw": "\"us-east-1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"Aggregate\"", + "got": "\"Seq", + "got_raw": "\"Seq Scan\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"Aggregate\"", + "got": "\"Seq", + "got_raw": "\"Seq Scan\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"new_value\"", + "got": "\"new_value\"", + "got_raw": "\"new_value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"text/plain\"", + "got": "\"application/json\"", + "got_raw": "\"application/json\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"text/plain\"", + "got": "\"application/json\"", + "got_raw": "\"application/json\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"test-owner\"", + "got": "\"test-owner\"", + "got_raw": "\"test-owner\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"test-owner\"", + "got": "\"test-owner\"", + "got_raw": "\"test-owner\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"Test Bucket\"", + "got": "\"Test Bucket\"", + "got_raw": "\"Test Bucket\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"Test Bucket\"", + "got": "\"Test Bucket\"", + "got_raw": "\"Test Bucket\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"my-app/0.0.1\"", + "got": "\"my-app/0.0.1\"", + "got_raw": "\"my-app/0.0.1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"my-app/0.0.1\"", + "got": "\"my-app/0.0.1\"", + "got_raw": "\"my-app/0.0.1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"my-app/0.0.1\"", + "got": "\"x-user-agent\"", + "got_raw": "\"x-user-agent\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"my-app/0.0.1\"", + "got": "\"x-user-agent\"", + "got_raw": "\"x-user-agent\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"my-app/0.0.1\"", + "got": "\"my-app/0.0.1\"", + "got_raw": "\"my-app/0.0.1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"Bearer s3cr3t\"", + "got": "\"Bearer s3cr3t\"", + "got_raw": "\"Bearer s3cr3t\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"Bearer s3cr3t\"", + "got": "\"Bearer s3cr3t\"", + "got_raw": "\"Bearer s3cr3t\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"/example_table\"", + "got": "\"/example_table\"", + "got_raw": "\"/example_table\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"x=like.%25a%25\"", + "got": "\"like(x", + "got_raw": "\"like(x, '%a%')\\n\"", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.43478260869565216 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"/example_table\"", + "got": "\"/example_table\"", + "got_raw": "\"/example_table\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"/example_table\"", + "got": "\"/example_table\"", + "got_raw": "\"/example_table\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"x=like.%25a%25\"", + "got": "\"like", + "got_raw": "\"like x '%a%'\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"/example_table\"", + "got": "\"/example_table\"", + "got_raw": "\"/example_table\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "b\"binary content\"", + "got": "mock_response.content", + "got_raw": "mock_response.content", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "supabase/supabase-py", + "expected": "b\"binary content\"", + "got": "mock_response.content", + "got_raw": "mock_response.content", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"x=lte.a&x=gte.b\"", + "got": "\"lte(x", + "got_raw": "\"lte(x, a)>e(x, b)\"", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.43478260869565216 + }, + { + "repo": "supabase/supabase-py", + "expected": "\"x=ilike.%25a%25\"", + "got": "\"ilike(x", + "got_raw": "\"ilike(x, '%a%')\\n\"", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.48 + }, + { + "repo": "supabase/supabase-py", + "expected": "Timeout(10)", + "got": "10.0", + "got_raw": "10.0", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "supabase/supabase-py", + "expected": "Timeout(30)", + "got": "Timeout(30)", + "got_raw": "Timeout(30)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "Timeout(10)", + "got": "10.0", + "got_raw": "10.0", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "supabase/supabase-py", + "expected": "Timeout(30)", + "got": "Timeout(30)", + "got_raw": "Timeout(30)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "info.keys()", + "got": "info[\"metadata\"]", + "got_raw": "info[\"metadata\"]", + "exact_match": false, + "code_bleu": 0.28744328957046744, + "edit_similarity": 0.37037037037037035 + }, + { + "repo": "supabase/supabase-py", + "expected": "info.keys()", + "got": "info[\"metadata\"]", + "got_raw": "info[\"metadata\"]", + "exact_match": false, + "code_bleu": 0.28744328957046744, + "edit_similarity": 0.37037037037037035 + }, + { + "repo": "supabase/supabase-py", + "expected": "Timeout(5.0)", + "got": "Timeout()", + "got_raw": "Timeout()", + "exact_match": false, + "code_bleu": 0.47920732818470435, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "supabase/supabase-py", + "expected": "Timeout(2.0)", + "got": "Timeout(2.0)", + "got_raw": "Timeout(2.0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "Timeout(2.0)", + "got": "Timeout(2.0)", + "got_raw": "Timeout(2.0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "supabase/supabase-py", + "expected": "headers.items()", + "got": "headers.items()", + "got_raw": "headers.items()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "OPTION_SCOPE_FLAG_MAP", + "got": "OPTION_SCOPE_FLAG_MAP", + "got_raw": "OPTION_SCOPE_FLAG_MAP", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "\"tmuxp\"", + "got": "session_name", + "got_raw": "session_name", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.10526315789473684 + }, + { + "repo": "tmux-python/libtmux", + "expected": "TEST_SESSION_NAME", + "got": "TEST_SESSION_NAME", + "got_raw": "TEST_SESSION_NAME", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "version", + "got": "version", + "got_raw": "version", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "expected_version", + "got": "expected_version", + "got_raw": "expected_version", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.BadSessionName)", + "got": "exc.TmuxpError)", + "got_raw": "exc.TmuxpError)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.LibTmuxException)", + "got": "exc.TmuxException)", + "got_raw": "exc.TmuxException)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.TmuxCommandNotFound)", + "got": "exc.TmuxCommandNotFound)", + "got_raw": "exc.TmuxCommandNotFound)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "\"test_2\"", + "got": "\"w_2\"", + "got_raw": "\"w_2\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "tmux-python/libtmux", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "50", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "tmux-python/libtmux", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "100", + "got": "\"100\"", + "got_raw": "\"100\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "tmux-python/libtmux", + "expected": "150", + "got": "150", + "got_raw": "150", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "[\"\"]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.39640237166757364, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "tmux-python/libtmux", + "expected": "True", + "got": "sparse[0]", + "got_raw": "sparse[0]", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "tmux-python/libtmux", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "50", + "got": "50", + "got_raw": "50", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "75", + "got": "75", + "got_raw": "75", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "\"$\"", + "got": "\"$", + "got_raw": "\"$ \"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.8 + }, + { + "repo": "tmux-python/libtmux", + "expected": "\"8\"", + "got": "\"10%\"", + "got_raw": "\"10%\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "tmux-python/libtmux", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "str(w[0].message)", + "got": "str(w[0].message)", + "got_raw": "str(w[0].message)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "\"status off\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "tmux-python/libtmux", + "expected": "server2.socket_name", + "got": "server2.socket_name", + "got_raw": "server2.socket_name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "88", + "got": "88", + "got_raw": "88", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "256", + "got": "256", + "got_raw": "256", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "cmd", + "got": "cmd", + "got_raw": "cmd", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "\"32\"", + "got": "\"32\"", + "got_raw": "\"32\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "\"HI\"", + "got": "\"HI\"", + "got_raw": "\"HI\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "v", + "got": "v", + "got_raw": "v", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "20", + "got": "\"20\"", + "got_raw": "\"20\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "tmux-python/libtmux", + "expected": "40", + "got": "40", + "got_raw": "40", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "\"2\"", + "got": "window.window_index", + "got_raw": "window.window_index", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "\"1\"", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "\"3\"", + "got": "\"0\"", + "got_raw": "\"0\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "tmux-python/libtmux", + "expected": "\"4\"", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "tmux-python/libtmux", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "pane", + "got": "pane", + "got_raw": "pane", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "t.cast(\"type[Exception]\", raises))", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.11939665909150465, + "edit_similarity": 0.2727272727272727 + }, + { + "repo": "tmux-python/libtmux", + "expected": "1", + "got": "len(session.windows)", + "got_raw": "len(session.windows)", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "v", + "got": "v", + "got_raw": "v", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "20", + "got": "20", + "got_raw": "20", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "40", + "got": "40", + "got_raw": "40", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "expected_result", + "got": "expected_result", + "got_raw": "expected_result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "version", + "got": "version", + "got_raw": "version", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "TMUX_NEXT_VERSION", + "got": "TMUX_NEXT_VERSION", + "got_raw": "TMUX_NEXT_VERSION", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.BadSessionName)", + "got": "exc.TmuxpError)", + "got_raw": "exc.TmuxpError)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.LibTmuxException)", + "got": "exc.TmuxVersionError)", + "got_raw": "exc.TmuxVersionError)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6190476190476191 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.TmuxCommandNotFound)", + "got": "exc.TmuxCommandNotFound)", + "got_raw": "exc.TmuxCommandNotFound)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "f\"{TMUX_MAX_VERSION}-master\"", + "got": "\"master\"", + "got_raw": "\"master\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "tmux-python/libtmux", + "expected": "f\"{TMUX_MAX_VERSION}-openbsd\"", + "got": "\"0.1.0\"", + "got_raw": "\"0.1.0\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.1111111111111111 + }, + { + "repo": "tmux-python/libtmux", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Pane\\.get\\(\\) was deprecated\")", + "got": "exc.DeprecatedError)", + "got_raw": "exc.DeprecatedError)", + "exact_match": false, + "code_bleu": 0.31426968052735443, + "edit_similarity": 0.5063291139240507 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Pane\\[key\\] lookup was deprecated\")", + "got": "exc.DeprecatedError)", + "got_raw": "exc.DeprecatedError)", + "exact_match": false, + "code_bleu": 0.31426968052735443, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Pane\\.resize_pane\\(\\) was deprecated\")", + "got": "DeprecatedError", + "got_raw": "DeprecatedError", + "exact_match": false, + "code_bleu": 0.0660670619445956, + "edit_similarity": 0.36585365853658536 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Pane\\.select_pane\\(\\) was deprecated\")", + "got": "DeprecatedError", + "got_raw": "DeprecatedError", + "exact_match": false, + "code_bleu": 0.0660670619445956, + "edit_similarity": 0.36585365853658536 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Pane\\.split_window\\(\\) was deprecated\")", + "got": "DeprecatedError", + "got_raw": "DeprecatedError", + "exact_match": false, + "code_bleu": 0.0660670619445956, + "edit_similarity": 0.3614457831325301 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Server\\.children was deprecated\")", + "got": "exc.DeprecatedError)", + "got_raw": "exc.DeprecatedError)", + "exact_match": false, + "code_bleu": 0.31426968052735443, + "edit_similarity": 0.4878048780487805 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Server\\.where\\(\\) was deprecated\")", + "got": "exc.DeprecatedError)", + "got_raw": "exc.DeprecatedError)", + "exact_match": false, + "code_bleu": 0.31426968052735443, + "edit_similarity": 0.4819277108433735 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Server\\._sessions was deprecated\")", + "got": "exc.DeprecatedError)", + "got_raw": "exc.DeprecatedError)", + "exact_match": false, + "code_bleu": 0.31426968052735443, + "edit_similarity": 0.4819277108433735 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Server\\.get_by_id\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4418604651162791 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Server\\.find_where\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4367816091954023 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Server\\.kill_server\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4318181818181818 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Server\\._list_panes\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4318181818181818 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Server\\.list_sessions\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4222222222222222 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Server\\._list_windows\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4222222222222222 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Server\\._update_panes\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4222222222222222 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Server\\._list_sessions\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4175824175824176 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Server\\._update_windows\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.41304347826086957 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Session\\.get\\(\\) was deprecated\")", + "got": "exc.DeprecatedError)", + "got_raw": "exc.DeprecatedError)", + "exact_match": false, + "code_bleu": 0.31426968052735443, + "edit_similarity": 0.4878048780487805 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Session\\.children was deprecated\")", + "got": "exc.DeprecatedError)", + "got_raw": "exc.DeprecatedError)", + "exact_match": false, + "code_bleu": 0.31426968052735443, + "edit_similarity": 0.4819277108433735 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Session\\._windows was deprecated\")", + "got": "exc.DeprecatedError)", + "got_raw": "exc.DeprecatedError)", + "exact_match": false, + "code_bleu": 0.31426968052735443, + "edit_similarity": 0.4819277108433735 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Session\\.where\\(\\) was deprecated\")", + "got": "exc.DeprecatedError)", + "got_raw": "exc.DeprecatedError)", + "exact_match": false, + "code_bleu": 0.31426968052735443, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Session\\[key\\] lookup was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4418604651162791 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Session\\.attached_pane was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4367816091954023 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Session\\.get_by_id\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4367816091954023 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Session\\.find_where\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4318181818181818 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Session\\.attached_window was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.42696629213483145 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Session\\.kill_session\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4222222222222222 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Session\\.list_windows\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4222222222222222 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Session\\._list_windows\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4175824175824176 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Session\\.attach_session\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.41304347826086957 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Server\\.where\\(\\) was deprecated\")", + "got": "exc.DeprecatedError)", + "got_raw": "exc.DeprecatedError)", + "exact_match": false, + "code_bleu": 0.31426968052735443, + "edit_similarity": 0.4819277108433735 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Window\\.where\\(\\) was deprecated\")", + "got": "exc.DeprecatedError)", + "got_raw": "exc.DeprecatedError)", + "exact_match": false, + "code_bleu": 0.31426968052735443, + "edit_similarity": 0.4819277108433735 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Session\\.where\\(\\) was deprecated\")", + "got": "exc.DeprecatedError)", + "got_raw": "exc.DeprecatedError)", + "exact_match": false, + "code_bleu": 0.31426968052735443, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Server\\.get_by_id\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4418604651162791 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Window\\.get_by_id\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4418604651162791 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Server\\.find_where\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4367816091954023 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Window\\.find_where\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4367816091954023 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Session\\.get_by_id\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4367816091954023 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Session\\.find_where\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4318181818181818 + }, + { + "repo": "tmux-python/libtmux", + "expected": "t.cast(\"type[Exception]\", raises))", + "got": "AssertionError)", + "got_raw": "AssertionError)", + "exact_match": false, + "code_bleu": 0.11939665909150465, + "edit_similarity": 0.32653061224489793 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Window\\._panes was deprecated\")", + "got": "exc.DeprecatedError)", + "got_raw": "exc.DeprecatedError)", + "exact_match": false, + "code_bleu": 0.31426968052735443, + "edit_similarity": 0.5 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Window\\.get\\(\\) was deprecated\")", + "got": "exc.DeprecatedError)", + "got_raw": "exc.DeprecatedError)", + "exact_match": false, + "code_bleu": 0.31426968052735443, + "edit_similarity": 0.49382716049382713 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Window\\.children was deprecated\")", + "got": "exc.DeprecatedError)", + "got_raw": "exc.DeprecatedError)", + "exact_match": false, + "code_bleu": 0.31426968052735443, + "edit_similarity": 0.4878048780487805 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Window\\.where\\(\\) was deprecated\")", + "got": "exc.DeprecatedError)", + "got_raw": "exc.DeprecatedError)", + "exact_match": false, + "code_bleu": 0.31426968052735443, + "edit_similarity": 0.4819277108433735 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Window\\[key\\] lookup was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4470588235294118 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Window\\.attached_pane was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4418604651162791 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Window\\.get_by_id\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4418604651162791 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Window\\.find_where\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4367816091954023 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Window\\.list_panes\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4367816091954023 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Window\\.kill_window\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4318181818181818 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Window\\._list_panes\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4318181818181818 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Window\\.split_window\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.42696629213483145 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.DeprecatedError, match=r\"Window\\.select_window\\(\\) was deprecated\")", + "got": "exc.DeprecatedError", + "got_raw": "exc.DeprecatedError", + "exact_match": false, + "code_bleu": 0.2802988050845715, + "edit_similarity": 0.4222222222222222 + }, + { + "repo": "tmux-python/libtmux", + "expected": "8", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "0.1", + "got": "0.1", + "got_raw": "0.1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "0.05", + "got": "0.1", + "got_raw": "0.1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "tmux-python/libtmux", + "expected": "\"libtmux_\"", + "got": "\"test-session-\"", + "got_raw": "\"test-session-\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.24 + }, + { + "repo": "tmux-python/libtmux", + "expected": "\"value1\"", + "got": "\"new_value1\"", + "got_raw": "\"new_value1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8 + }, + { + "repo": "tmux-python/libtmux", + "expected": "\"value2\"", + "got": "\"value2\"", + "got_raw": "\"value2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "os.environ", + "got": "os.environ", + "got_raw": "os.environ", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "\"new_value\"", + "got": "\"new_value\"", + "got_raw": "\"new_value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "\"new_value1\"", + "got": "\"new_value1\"", + "got_raw": "\"new_value1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "\"original_value\"", + "got": "\"original_value\"", + "got_raw": "\"original_value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "8", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "rng", + "got": "rng", + "got_raw": "rng", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "result2", + "got": "result2", + "got_raw": "result2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "third_name", + "got": "second_name", + "got_raw": "second_name", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "tmux-python/libtmux", + "expected": "end - ini", + "got": "end - ini)", + "got_raw": "end - ini)", + "exact_match": false, + "code_bleu": 0.668740304976422, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "tmux-python/libtmux", + "expected": "exc.WaitTimeout)", + "got": "exc.RetryTimeout)", + "got_raw": "exc.RetryTimeout)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7878787878787878 + }, + { + "repo": "tmux-python/libtmux", + "expected": "expected_value", + "got": "expected_value", + "got_raw": "expected_value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "test_case.expected", + "got": "test_case.expected", + "got_raw": "test_case.expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "test_case.expected_keys", + "got": "sorted(test_case.initial_adds", + "got_raw": "sorted(test_case.initial_adds + test_case.append_values)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.5 + }, + { + "repo": "tmux-python/libtmux", + "expected": "test_case.expected_values", + "got": "test_case.adds", + "got_raw": "test_case.adds", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "tmux-python/libtmux", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "window_name", + "got": "window_name", + "got_raw": "window_name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "session_name", + "got": "session_name", + "got_raw": "session_name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/libtmux", + "expected": "RuntimeError)", + "got": "test_error)", + "got_raw": "test_error)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5833333333333334 + }, + { + "repo": "tmux-python/libtmux", + "expected": "initial_windows", + "got": "initial_windows", + "got_raw": "initial_windows", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "6", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "1", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "RDSError)", + "got": "RDSError)", + "got_raw": "RDSError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "str(error)", + "got": "str(error)", + "got_raw": "str(error)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "[ call(ANY), call(ANY), ]", + "got": "[call(\"test\"), call(\"test\")]", + "got_raw": "[call(\"test\"), call(\"test\")]", + "exact_match": false, + "code_bleu": 0.4109771142019087, + "edit_similarity": 0.6037735849056604 + }, + { + "repo": "toluaina/pgsync", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"book\"", + "got": "\"book\"", + "got_raw": "\"book\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "tables", + "got": "tables", + "got_raw": "tables", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "[\"id\"]", + "got": "\"publisher\"", + "got_raw": "\"publisher\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"city\"", + "got": "\"city\"", + "got_raw": "\"city\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"123\"", + "got": "\"123\"", + "got_raw": "\"123\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"test\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"value\"", + "got": "\"value\"", + "got_raw": "\"value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "6", + "got": "6", + "got_raw": "6", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "2002", + "got": "2002", + "got_raw": "2002", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "[1, 2]", + "got": "[1, 2]", + "got_raw": "[1, 2]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "[3, 4, 5]", + "got": "[3, 4, 5]", + "got_raw": "[3, 4, 5]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "{2001, 2003}", + "got": "{2001, 2003}", + "got_raw": "{2001, 2003}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"pop size: 2\")", + "got": "\"pop: %s\"", + "got_raw": "\"pop: %s\", items)", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"pop size: 3\")", + "got": "\"pop size: 3\")", + "got_raw": "\"pop size: 3\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"foo:something\"", + "got": "\"foo:something\"", + "got_raw": "\"foo:something\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "ConnectionError)", + "got": "ConnectionError)", + "got_raw": "ConnectionError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "{3001, 3002, 3003}", + "got": "{3001, 3002, 3003}", + "got_raw": "{3001, 3002, 3003}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "100", + "got": "100", + "got_raw": "100", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "2.0", + "got": "2.0", + "got_raw": "2.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "url)", + "got": "_search_client.assert_not_called()", + "got_raw": "_search_client.assert_not_called()", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.15789473684210525 + }, + { + "repo": "toluaina/pgsync", + "expected": "60.0", + "got": "60.0", + "got_raw": "60.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "Exception)", + "got": "Exception)", + "got_raw": "Exception)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"OpenSearch\"", + "got": "\"opensearch\"", + "got_raw": "\"opensearch\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "toluaina/pgsync", + "expected": "RuntimeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "toluaina/pgsync", + "expected": "str(excinfo.value)", + "got": "str(excinfo)", + "got_raw": "str(excinfo)", + "exact_match": false, + "code_bleu": 0.4714045207910316, + "edit_similarity": 0.8 + }, + { + "repo": "toluaina/pgsync", + "expected": "url", + "got": "url", + "got_raw": "url", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"http://some-domain:33\"", + "got": "\"some-domain:33\"", + "got_raw": "\"some-domain:33\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8205128205128205 + }, + { + "repo": "toluaina/pgsync", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "4", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "3", + "got": "4", + "got_raw": "4", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "val", + "got": "val", + "got_raw": "val", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "nodes)", + "got": "\"testdb\")", + "got_raw": "\"testdb\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4 + }, + { + "repo": "toluaina/pgsync", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "nodes)", + "got": "docs)", + "got_raw": "docs)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "toluaina/pgsync", + "expected": "docs[0][\"_id\"]", + "got": "docs[0][\"doc_id\"]", + "got_raw": "docs[0][\"doc_id\"]", + "exact_match": false, + "code_bleu": 0.7071067811865475, + "edit_similarity": 0.9032258064516129 + }, + { + "repo": "toluaina/pgsync", + "expected": "docs[1][\"_id\"]", + "got": "docs[1][\"_id\"]", + "got_raw": "docs[1][\"_id\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "docs[2][\"_id\"]", + "got": "docs[2][\"_id\"]", + "got_raw": "docs[2][\"_id\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "docs[0][\"_source\"]", + "got": "docs[0]", + "got_raw": "docs[0]", + "exact_match": false, + "code_bleu": 0.5714285714285714, + "edit_similarity": 0.56 + }, + { + "repo": "toluaina/pgsync", + "expected": "str(excinfo.value)", + "got": "str(excinfo)", + "got_raw": "str(excinfo)", + "exact_match": false, + "code_bleu": 0.4714045207910316, + "edit_similarity": 0.8 + }, + { + "repo": "toluaina/pgsync", + "expected": "NodeAttributeError)", + "got": "NodeAttributeError)", + "got_raw": "NodeAttributeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "ColumnNotFoundError)", + "got": "ColumnNotFoundError)", + "got_raw": "ColumnNotFoundError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "doc.get(\"node\", {}))", + "got": "\"testdb\")", + "got_raw": "\"testdb\")", + "exact_match": false, + "code_bleu": 0.1074569931823542, + "edit_similarity": 0.27586206896551724 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"abc\"", + "got": "\"abc\"", + "got_raw": "\"abc\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"def\"", + "got": "\"def\"", + "got_raw": "\"def\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"ghi\"", + "got": "\"ghi\"", + "got_raw": "\"ghi\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "nodes)", + "got": "docs)", + "got_raw": "docs)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"abc\"", + "got": "\"abc\"", + "got_raw": "\"abc\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"def\"", + "got": "\"def\"", + "got_raw": "\"def\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"ghi\"", + "got": "\"ghi\"", + "got_raw": "\"ghi\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "nodes)", + "got": "docs)", + "got_raw": "docs)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "toluaina/pgsync", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "123", + "got": "123", + "got_raw": "123", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "{\"id\": \"my_id\"}", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.23784142300054423, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "toluaina/pgsync", + "expected": "{ \"id\": 1, \"code\": \"ABC\", }", + "got": "{\"id\": 1, \"code\": \"\"}", + "got_raw": "{\"id\": 1, \"code\": \"\"}", + "exact_match": false, + "code_bleu": 0.7042880610329794, + "edit_similarity": 0.875 + }, + { + "repo": "toluaina/pgsync", + "expected": "{ \"id\": 1, \"my_code\": None, }", + "got": "{ \"id\": 1, \"my_code\": None, }", + "got_raw": "{ \"id\": 1, \"my_code\": None, }", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "{ \"id\": 1, \"code\": \"ABCDEF\", }", + "got": "row", + "got_raw": "row", + "exact_match": false, + "code_bleu": 0.05, + "edit_similarity": 0.06060606060606061 + }, + { + "repo": "toluaina/pgsync", + "expected": "{ \"user\": {\"id\": [1, 2, 3]}, }", + "got": "[\"user_id\"]", + "got_raw": "[\"user_id\"]", + "exact_match": false, + "code_bleu": 0.09374999999999999, + "edit_similarity": 0.43902439024390244 + }, + { + "repo": "toluaina/pgsync", + "expected": "{ \"child_table\": {\"name\": \"child_name\"}, }", + "got": "{ \"table\": \"parent\", \"columns\": [\"id\"], \"children\": [ { \"table\": \"child_table\", \"columns\": [\"name\"], \"relationship\": { \"variant\": \"object\", \"type\": \"one_to_one\", }, \"transform\": { \"rename\": { \"name\": \"child_name\", }, }, }, ], }", + "got_raw": "{ \"table\": \"parent\", \"columns\": [\"id\"], \"children\": [ { \"table\": \"child_table\", \"columns\": [\"name\"], \"relationship\": { \"variant\": \"object\", \"type\": \"one_to_one\", }, \"transform\": { \"rename\": { \"name\": \"child_name\", }, }, }, ], }", + "exact_match": false, + "code_bleu": 0.10320057204811237, + "edit_similarity": 0.31226765799256506 + }, + { + "repo": "toluaina/pgsync", + "expected": "{ \"id\": 1, \"codes\": [\"A=B\", \"C=D\", \"E=F\"], }", + "got": "{ \"id\": 1, \"codes\": [\"A=\", \"C=\", \"E=\"], }", + "got_raw": "{ \"id\": 1, \"codes\": [\"A=\", \"C=\", \"E=\"], }", + "exact_match": false, + "code_bleu": 0.5856596027429395, + "edit_similarity": 0.9647058823529412 + }, + { + "repo": "toluaina/pgsync", + "expected": "{ \"first\": \"A\", \"second\": \"B\", \"combined\": \"AB\", }", + "got": "{\"first\": \"A\", \"second\": \"B\", \"combined\": \"A B\"}", + "got_raw": "{\"first\": \"A\", \"second\": \"B\", \"combined\": \"A B\"}", + "exact_match": false, + "code_bleu": 0.7944171725947274, + "edit_similarity": 0.9591836734693877 + }, + { + "repo": "toluaina/pgsync", + "expected": "{ \"code\": {\"@\": \"", + "got": "{ \"replace\": { \"code\": {\"@\": \"", + "got_raw": "{ \"replace\": { \"code\": {\"@\": \"#\"}, } }", + "exact_match": false, + "code_bleu": 0.6786502681586726, + "edit_similarity": 0.723404255319149 + }, + { + "repo": "toluaina/pgsync", + "expected": "{ \"id\": 1, \"code\": \"ABC=DEF GHI\", \"name\": \"unchanged\", }", + "got": "{ \"id\": 1, \"code\": \"ABC DEF GHI\", \"name\": \"unchanged\", }", + "got_raw": "{ \"id\": 1, \"code\": \"ABC DEF GHI\", \"name\": \"unchanged\", }", + "exact_match": false, + "code_bleu": 0.8003203203844999, + "edit_similarity": 0.9821428571428571 + }, + { + "repo": "toluaina/pgsync", + "expected": "{ \"id\": 1, \"first\": \"A\", \"combined\": \"A-second-third\", }", + "got": "{\"id\": 1, \"first\": \"A\", \"second\": \"null\", \"third\": \"null\", \"combined\": \"A-null-null\"}", + "got_raw": "{\"id\": 1, \"first\": \"A\", \"second\": \"null\", \"third\": \"null\", \"combined\": \"A-null-null\"}", + "exact_match": false, + "code_bleu": 0.48485170206034506, + "edit_similarity": 0.6099290780141844 + }, + { + "repo": "toluaina/pgsync", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "primary_keys", + "got": "primary_keys", + "got_raw": "primary_keys", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "foreign_keys", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "sorted(foreign_keys)", + "got": "foreign_keys", + "got_raw": "foreign_keys", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.75 + }, + { + "repo": "toluaina/pgsync", + "expected": "nodes)", + "got": "data)", + "got_raw": "data)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "toluaina/pgsync", + "expected": "[ { \"_id\": \"abc\", \"_index\": \"testdb\", \"_source\": { \"_meta\": { \"book\": {\"isbn\": [\"abc\"]}, \"contact\": {\"id\": [1, 2]}, \"user\": {\"id\": [1, 2]}, }, \"buyer\": { \"contacts\": [ { \"contact_items\": None, \"id\": 1, \"name\": \"Contact 1\", } ], \"id\": 1, \"name\": \"Fonzy Bear\", }, \"description\": \"Tigers are fierce creatures\", \"isbn\": \"abc\", \"seller\": { \"contacts\": [ { \"contact_items\": None, \"id\": 2, \"name\": \"Contact 2\", } ], \"id\": 2, \"name\": \"Jack Jones\", }, \"title\": \"The Tiger Club\", }, } ]", + "got": "[", + "got_raw": "[", + "exact_match": false, + "code_bleu": 0.004504572405313338, + "edit_similarity": 0.0041928721174004195 + }, + { + "repo": "toluaina/pgsync", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "str(excinfo.value)", + "got": "\"Missing", + "got_raw": "\"Missing config\"", + "exact_match": false, + "code_bleu": 0.15060033393497413, + "edit_similarity": 0.23076923076923078 + }, + { + "repo": "toluaina/pgsync", + "expected": "FileNotFoundError)", + "got": "FileNotFoundError)", + "got_raw": "FileNotFoundError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"http://localhost:9999\"", + "got": "get_search_url())", + "got_raw": "get_search_url())", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.1 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"redis://skynet:6379/0\"", + "got": "\"redis://skynet:6379/0\"", + "got_raw": "\"redis://skynet:6379/0\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"redis://localhost:6379/0\"", + "got": "\"redis://localhost:6379\"", + "got_raw": "\"redis://localhost:6379\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.96 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"redis://:1234@localhost:9999/0\"", + "got": "\"redis://localhost:6379/0\"", + "got_raw": "\"redis://localhost:6379/0\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7931034482758621 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"redis://kermit:1234@localhost:6379/0\"", + "got": "\"redis://user:password@localhost:6379\"", + "got_raw": "\"redis://user:password@localhost:6379\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"redis://kermit:1234@localhost:9999/0\"", + "got": "\"redis://kermit:1234@localhost:6379/0\"", + "got_raw": "\"redis://kermit:1234@localhost:6379/0\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9210526315789473 + }, + { + "repo": "toluaina/pgsync", + "expected": "f\"http://localhost:{ELASTICSEARCH_PORT}\"", + "got": "\"localhost\"", + "got_raw": "\"localhost\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.43137254901960786 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"Connecting to Redis without password.\")", + "got": "_logger.warning.assert_called_once_with(\"Redis URL is not set\")", + "got_raw": "_logger.warning.assert_called_once_with(\"Redis URL is not set\")", + "exact_match": false, + "code_bleu": 0.16036590969929357, + "edit_similarity": 0.3883495145631068 + }, + { + "repo": "toluaina/pgsync", + "expected": "f\"https://localhost:{ELASTICSEARCH_PORT}\"", + "got": "\"search\"", + "got_raw": "\"search\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.20408163265306123 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"mysql+pymysql://kermit:12345@localhost:3306/mydb\"", + "got": "\"mysql\"", + "got_raw": "\"mysql\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.24561403508771928 + }, + { + "repo": "toluaina/pgsync", + "expected": "f\"http://kermit:1234@localhost:{ELASTICSEARCH_PORT}\"", + "got": "\"search\"", + "got_raw": "\"search\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.1 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"postgresql+psycopg2://kermit:12345@localhost:5432/mydb\"", + "got": "\"postgresql+psycopg2://postgres:postgres@localhost:5432/mydb\"", + "got_raw": "\"postgresql+psycopg2://postgres:postgres@localhost:5432/mydb\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7863247863247863 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"Connecting to Redis with custom username and password.\")", + "got": "_logger.warning.assert_called_once_with(\"No password provided\")", + "got_raw": "_logger.warning.assert_called_once_with(\"No password provided\")", + "exact_match": false, + "code_bleu": 0.16036590969929357, + "edit_similarity": 0.4 + }, + { + "repo": "toluaina/pgsync", + "expected": "str( excinfo.value )", + "got": "str(excinfo.value)", + "got_raw": "str(excinfo.value)", + "exact_match": false, + "code_bleu": 1.0, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "toluaina/pgsync", + "expected": "5", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "15", + "got": "15", + "got_raw": "15", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "url", + "got": "url", + "got_raw": "url", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "\"0\"", + "got": "\"0\"", + "got_raw": "\"0\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "ddl", + "got": "ddl", + "got_raw": "ddl", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "toluaina/pgsync", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "0.5", + "got": "0.5", + "got_raw": "0.5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "0.3", + "got": "0.3", + "got_raw": "0.3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"run_sql\"", + "got": "\"run_sql\"", + "got_raw": "\"run_sql\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"Show me top customers\"", + "got": "\"Show me top customers\"", + "got_raw": "\"Show me top customers\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "100", + "got": "100", + "got_raw": "100", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "0.95", + "got": "0.95", + "got_raw": "0.95", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "0.88", + "got": "0.88", + "got_raw": "0.88", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "0.8", + "got": "0.8", + "got_raw": "0.8", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"auto\"", + "got": "\"test_tool\"", + "got_raw": "\"test_tool\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"gpt-5\"", + "got": "\"gpt-5\"", + "got_raw": "\"gpt-5\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "payload", + "got": "payload", + "got_raw": "payload", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"gpt-4o\"", + "got": "\"gpt-4o\"", + "got_raw": "\"gpt-4o\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"system\"", + "got": "\"system\"", + "got_raw": "\"system\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"env-key\"", + "got": "\"env-key\"", + "got_raw": "\"env-key\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"test-key\"", + "got": "\"test-key\"", + "got_raw": "\"test-key\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"function\"", + "got": "\"object\"", + "got_raw": "\"object\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "vanna-ai/vanna", + "expected": "call_kwargs", + "got": "call_kwargs", + "got_raw": "call_kwargs", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"test_tool\"", + "got": "\"test_tool\"", + "got_raw": "\"test_tool\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"2024-10-21\"", + "got": "\"2024-10-21\"", + "got_raw": "\"2024-10-21\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"2024-06-01\"", + "got": "\"2024-06-01\"", + "got_raw": "\"2024-06-01\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"new_collection\"", + "got": "\"new_collection\"", + "got_raw": "\"new_collection\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"test_collection\"", + "got": "\"test_collection\"", + "got_raw": "\"test_collection\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "[\"test question 1\", \"test question 2\"]", + "got": "[\"test question 1\", \"test question 2\"]", + "got_raw": "[\"test question 1\", \"test question 2\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "0.7", + "got": "0.7", + "got_raw": "0.7", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"gemini-2.5-pro\"", + "got": "\"gemini-2.5-pro\"", + "got_raw": "\"gemini-2.5-pro\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "response.content", + "got": "response.content", + "got_raw": "response.content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "ValueError, match=\"Google API key is required\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.3793103448275862 + }, + { + "repo": "vanna-ai/vanna", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"What are the total sales?\"", + "got": "\"What are the total sales?\"", + "got_raw": "\"What are the total sales?\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"SELECT SUM(Total) FROM Invoice\"", + "got": "\"SELECT SUM(Total) FROM Invoice\"", + "got_raw": "\"SELECT SUM(Total) FROM Invoice\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"The Invoice table contains all sales transactions.\"", + "got": "\"Text memory saved\"", + "got_raw": "\"Text memory saved\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.30985915492957744 + }, + { + "repo": "vanna-ai/vanna", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "call", + "got": "call[\"system_prompt\"]", + "got_raw": "call[\"system_prompt\"]", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.32 + }, + { + "repo": "vanna-ai/vanna", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"test_user\"", + "got": "\"123\"", + "got_raw": "\"123\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.25 + }, + { + "repo": "vanna-ai/vanna", + "expected": "first_request.system_prompt", + "got": "first_request.system_prompt", + "got_raw": "first_request.system_prompt", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "content", + "got": "content", + "got_raw": "content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"success\"", + "got": "\"Success\"", + "got_raw": "\"Success\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "vanna-ai/vanna", + "expected": "content.lower()", + "got": "content", + "got_raw": "content", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.6363636363636364 + }, + { + "repo": "vanna-ai/vanna", + "expected": "ComponentType.CARD", + "got": "ComponentType.CARD", + "got_raw": "ComponentType.CARD", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "status.message.lower()", + "got": "status.message", + "got_raw": "status.message", + "exact_match": false, + "code_bleu": 0.36038417796587763, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "vanna-ai/vanna", + "expected": "user_result.result_for_llm", + "got": "user_result.result_for_llm", + "got_raw": "user_result.result_for_llm", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "0.5", + "got": "0.5", + "got_raw": "0.5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "4096", + "got": "4096", + "got_raw": "4096", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"user\"", + "got": "\"user\"", + "got_raw": "\"user\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"llama3.2\"", + "got": "\"llama3.2\"", + "got_raw": "\"llama3.2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "response.content", + "got": "response.content", + "got_raw": "response.content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "\"http://localhost:11434\"", + "got": "\"http://localhost:11434\"", + "got_raw": "\"http://localhost:11434\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "vanna-ai/vanna", + "expected": "4", + "got": "5", + "got_raw": "5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "\"new.value\"", + "got": "\"new.value\"", + "got_raw": "\"new.value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "AttributeError)", + "got": "ImproperlyConfigured)", + "got_raw": "ImproperlyConfigured)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "DEFAULTS[\"INIT_API_FN\"]", + "got": "DEFAULTS[\"INIT_API_FN\"]", + "got_raw": "DEFAULTS[\"INIT_API_FN\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "\"custom.module.init_api\"", + "got": "\"custom.module.init_api\"", + "got_raw": "\"custom.module.init_api\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "3", + "got": "5", + "got_raw": "5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "VERSION", + "got": "VERSION", + "got_raw": "VERSION", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "thread.id", + "got": "thread.id", + "got_raw": "thread.id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "\"New name\"", + "got": "\"New name\"", + "got_raw": "\"New name\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "PACKAGE_NAME", + "got": "PACKAGE_NAME", + "got_raw": "PACKAGE_NAME", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "assistant_id", + "got": "assistant_id", + "got_raw": "assistant_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "30", + "got": "30", + "got_raw": "30", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "2001", + "got": "2001", + "got_raw": "2001", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "\"John\"", + "got": "\"John\"", + "got_raw": "\"John\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "\"Shrek\"", + "got": "\"Shrek\"", + "got_raw": "\"Shrek\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "\"system\"", + "got": "\"response\"", + "got_raw": "\"response\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "call_kwargs", + "got": "call_kwargs", + "got_raw": "call_kwargs", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "[\"Animation\", \"Comedy\"]", + "got": "[\"Animation\", \"Comedy\"]", + "got_raw": "[\"Animation\", \"Comedy\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "[m[\"data\"][\"id\"] for m in expected_messages]", + "got": "[ m[\"data\"][\"id\"] for m in expected_messages ]", + "got_raw": "[ m[\"data\"][\"id\"] for m in expected_messages ]", + "exact_match": false, + "code_bleu": 1.0, + "edit_similarity": 0.9777777777777777 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "\"You are a temperature bot. Today is 2024-06-09.\"", + "got": "expected_messages[0])", + "got_raw": "expected_messages[0])", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.22857142857142856 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "\"11 W 53rd St, New York, NY 10019, United States.\"", + "got": "\"11 W 53rd St, New York, NY 10019, United States.\"", + "got_raw": "\"11 W 53rd St, New York, NY 10019, United States.\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "{\"input\": \"What is the temperature today in Recife?\"})", + "got": "", + "got_raw": "", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "\"I'm at Central Park W & 79st, New York, NY 10024, United States.\"", + "got": "\"input\"", + "got_raw": "\"input\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.1095890410958904 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "\"You're right by the Museum of Modern Art (MoMA), which houses an impressive collection of modern and contemporary art, including works by Van Gogh and Warhol. Enjoy exploring the vibrant gallery spaces and temporary exhibitions. It's a must-visit for art enthusiasts!\"", + "got": "response_1[\"output\"]", + "got_raw": "response_1[\"output\"]", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.01384083044982699 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "\"You're right next to the American Museum of Natural History, where you can explore fascinating exhibits about dinosaurs, space, and human cultures. Additionally, enjoy a stroll through Central Park to experience its beautiful landscapes and iconic landmarks like Belvedere Castle and Bow Bridge. Don't miss the opportunity to visit both these highlights!\"", + "got": "\"35 degrees Celsius\"", + "got_raw": "\"35 degrees Celsius\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.026595744680851064 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "[ \"tool_d\", \"tool_c\", \"tool_b\", \"tool_a\", ]", + "got": "[\"a\", \"b\", \"c\", \"d\"]", + "got_raw": "[\"a\", \"b\", \"c\", \"d\"]", + "exact_match": false, + "code_bleu": 0.16717552199220237, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "[ m[\"data\"].get(attr) for m in expected_messages ]", + "got": "[None]", + "got_raw": "[None]", + "exact_match": false, + "code_bleu": 0.09999999999999999, + "edit_similarity": 0.17857142857142858 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "\"Hello\"", + "got": "\"Hello\"", + "got_raw": "\"Hello\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "\"\"", + "got": "\"temperature_assistant\"", + "got_raw": "\"temperature_assistant\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.16 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "user", + "got": "user", + "got_raw": "user", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "thread", + "got": "thread", + "got_raw": "thread", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "\"My thread\"", + "got": "\"My thread\"", + "got_raw": "\"My thread\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "assistant_id", + "got": "assistant_id", + "got_raw": "assistant_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "\"My updated thread\"", + "got": "\"Updated\"", + "got_raw": "\"Updated\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "vintasoftware/django-ai-assistant", + "expected": "AIUserNotAllowedError)", + "got": "AIUserNotAllowedError)", + "got_raw": "AIUserNotAllowedError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "px_terminal.red(\" apa \") + px_terminal.yellow(\" b\") + px_terminal.blue(\" \") + px_terminal.inverse_video(\" \")", + "got": "[(\"apa\"", + "got_raw": "[(\"apa\", 1000.0), (\"bepa\", 300.0), (\"cepa\", 50.0)]", + "exact_match": false, + "code_bleu": 0.05020011131165804, + "edit_similarity": 0.10434782608695652 + }, + { + "repo": "walles/px", + "expected": "px_terminal.red(\" \u00e5pa \") + px_terminal.yellow(\" b\") + px_terminal.blue(\" \") + px_terminal.inverse_video(\" \")", + "got": "\"\u00e5pa 1000.0\\nb\u00e4pa 300.0\\n\"", + "got_raw": "\"\u00e5pa 1000.0\\nb\u00e4pa 300.0\\n\"", + "exact_match": false, + "code_bleu": 0.018518518518518517, + "edit_similarity": 0.1791044776119403 + }, + { + "repo": "walles/px", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "5", + "got": "px_commandline.coalesce_count([\"/a\"", + "got_raw": "px_commandline.coalesce_count([\"/a\", \"b\", \"c\", \"d\", \"e\"], exists=exists)", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "\"\ud83d\ude00\"", + "got": "", + "got_raw": "", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "\"sh\"", + "got": "\"sh\"", + "got_raw": "\"sh\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "\"go\"", + "got": "\"go\"", + "got_raw": "\"go\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "\"hej\"", + "got": "\"hej\"", + "got_raw": "\"hej\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "\"mod\"", + "got": "\"python\"", + "got_raw": "\"python\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "walles/px", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "1)", + "got": "2)", + "got_raw": "2)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "walles/px", + "expected": "4)", + "got": "4)", + "got_raw": "4)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "8)", + "got": "8)", + "got_raw": "8)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "96)", + "got": "96)", + "got_raw": "96)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "(1, 2)", + "got": "(1, 2)", + "got_raw": "(1, 2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "physical", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "(12, 34)", + "got": "(12, 34)", + "got_raw": "(12, 34)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "(None, None)", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.18072040072196896, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "expected_logical", + "got": "expected_logical", + "got_raw": "expected_logical", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "expected_physical", + "got": "expected_physical", + "got_raw": "expected_physical", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "\"/\"", + "got": "\"/\"", + "got_raw": "\"/\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "[friend]", + "got": "[friend]", + "got_raw": "[friend]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "[p1, p2]", + "got": "[p1, p2]", + "got_raw": "[p1, p2]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "\"/notroot\"", + "got": "\"/notroot\"", + "got_raw": "\"/notroot\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "[\"a\", \"b\"]", + "got": "[\"a\", \"b\"]", + "got_raw": "[\"a\", \"b\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "[\"a\", \"-b\", \"c\"]", + "got": "[\"a\", \"c\"]", + "got_raw": "[\"a\", \"c\"]", + "exact_match": false, + "code_bleu": 0.5050762722761053, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "walles/px", + "expected": "\"\"", + "got": "-1", + "got_raw": "-1", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "5", + "got": "7", + "got_raw": "7", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "123", + "got": "123", + "got_raw": "123", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "\"/\"", + "got": "\"a\"", + "got_raw": "\"a\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "walles/px", + "expected": "\"r\"", + "got": "\"r\"", + "got_raw": "\"r\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "100", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "walles/px", + "expected": "None", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "\"localhost:ipp\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "walles/px", + "expected": "\"localhost:63342\"", + "got": "\"p123\"", + "got_raw": "\"p123\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2608695652173913 + }, + { + "repo": "walles/px", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "4", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "{25}", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.30120066786994826, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "set()", + "got": "set()", + "got_raw": "set()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "{200}", + "got": "[files[1].pid]", + "got_raw": "[files[1].pid]", + "exact_match": false, + "code_bleu": 0.13485111859503685, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "{100}", + "got": "{100}", + "got_raw": "{100}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "\"Johan\"", + "got": "\"Name", + "got_raw": "\"Name \\\"Johan\\\"\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "walles/px", + "expected": "{25, 26}", + "got": "(25, 26)", + "got_raw": "(25, 26)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.75 + }, + { + "repo": "walles/px", + "expected": "\"PID 45\"", + "got": "\"45\"", + "got_raw": "\"45\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "walles/px", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "[l2, l1]", + "got": "[l2, l1]", + "got_raw": "[l2, l1]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "[(\"a\", 5)]", + "got": "[(\"a\", 0)]", + "got_raw": "[(\"a\", 0)]", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.9 + }, + { + "repo": "walles/px", + "expected": "[(\"a\", 0), (\"b\", 6), (\"c\", 5)]", + "got": "l2", + "got_raw": "l2", + "exact_match": false, + "code_bleu": 0.02631578947368421, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "[process_other_pid, process_other_starttime]", + "got": "[process_identical]", + "got_raw": "[process_identical]", + "exact_match": false, + "code_bleu": 0.29999999999999993, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "walles/px", + "expected": "[ (\"a\", 0), (\"b\", 0), (\"c\", 5), ]", + "got": "[(\"a\", \"b\", \"c\"), (\"a\", \"b\", \"c\"), (\"a\", \"b\", \"c\"), (\"a\", \"b\", \"c\"), (\"a\", \"b\", \"c\")]", + "got_raw": "[(\"a\", \"b\", \"c\"), (\"a\", \"b\", \"c\"), (\"a\", \"b\", \"c\"), (\"a\", \"b\", \"c\"), (\"a\", \"b\", \"c\")]", + "exact_match": false, + "code_bleu": 0.16088986597586105, + "edit_similarity": 0.423728813559322 + }, + { + "repo": "walles/px", + "expected": "[ \"init -> \" + px_terminal.bold(\"iTerm\") + \"(1) -> \" + px_terminal.bold(\"fish\") + \"(1)\" ]", + "got": "[\"init\"", + "got_raw": "[\"init\", \"iTerm\", \"fish\"]", + "exact_match": false, + "code_bleu": 0.051169996753502, + "edit_similarity": 0.14583333333333334 + }, + { + "repo": "walles/px", + "expected": "[ \"init -> \" + px_terminal.bold(\"iTerm\") + \"(3) -> \" + px_terminal.bold(\"fish\") + \"(2)\" ]", + "got": "[", + "got_raw": "[\n \"Launches:\",\n \" init: 2\",\n \" init:iTerm: 3\",\n \" init:iTerm:fish: 2\",\n ]", + "exact_match": false, + "code_bleu": 0.028314455119112403, + "edit_similarity": 0.022222222222222223 + }, + { + "repo": "walles/px", + "expected": "{ \"init -> \" + px_terminal.bold(\"iTerm\") + \"(3)\", \"init -> \" + px_terminal.bold(\"(i)Term\") + \"(1)\", \"init -> \" + px_terminal.bold(\"i(Term)\") + \"(1)\", \"init -> \" + px_terminal.bold(\"iTerm()\") + \"(1)\", }", + "got": "set([\"iTerm\", \"iTerm\", \"iTerm\", \"iTerm\"])", + "got_raw": "set([\"iTerm\", \"iTerm\", \"iTerm\", \"iTerm\"])", + "exact_match": false, + "code_bleu": 0.03707392811274897, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "8", + "got": "15", + "got_raw": "15", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "(0, 0, 0, 1.0)", + "got": "(0.0, 0.0, 0.0, 0.0)", + "got_raw": "(0.0, 0.0, 0.0, 0.0)", + "exact_match": false, + "code_bleu": 0.18575057999133596, + "edit_similarity": 0.6470588235294118 + }, + { + "repo": "walles/px", + "expected": "(0, 1, 3, 1.0)", + "got": "(1.0 / 6.0 - 0.01, 1.0 / 6.0 + 0.01, 1.0, 1.0)", + "got_raw": "(1.0 / 6.0 - 0.01, 1.0 / 6.0 + 0.01, 1.0, 1.0)", + "exact_match": false, + "code_bleu": 0.15464260451973766, + "edit_similarity": 0.4 + }, + { + "repo": "walles/px", + "expected": "(2, 3, 3, 1.0)", + "got": "(3, 4, 5, 1.0)", + "got_raw": "(3, 4, 5, 1.0)", + "exact_match": false, + "code_bleu": 0.3356891925037239, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "walles/px", + "expected": "px_load.get_load_string((0.3, 0.2, 0.1))", + "got": "px_load(\"0.3\")", + "got_raw": "px_load(\"0.3\")", + "exact_match": false, + "code_bleu": 0.14242900213207801, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "walles/px", + "expected": "px_load.get_load_string((3.0, 0.2, 0.1))", + "got": "px_load.get_load_string((3.0, 2.0, 1.0))", + "got_raw": "px_load.get_load_string((3.0, 2.0, 1.0))", + "exact_match": false, + "code_bleu": 0.6262844962765468, + "edit_similarity": 0.9 + }, + { + "repo": "walles/px", + "expected": "px_load.get_load_string((2.0, 3.0, 4.0))", + "got": "px_load.get_load_string((2.0, 0.2, 0.1))", + "got_raw": "px_load.get_load_string((2.0, 0.2, 0.1))", + "exact_match": false, + "code_bleu": 0.6262844962765468, + "edit_similarity": 0.9 + }, + { + "repo": "walles/px", + "expected": "px_load.get_load_string((1.135135, 0.2, 0.1))", + "got": "px_load.get_load_string((1.1, 0.2, 0.1))", + "got_raw": "px_load.get_load_string((1.1, 0.2, 0.1))", + "exact_match": false, + "code_bleu": 0.7611606003349892, + "edit_similarity": 0.9411764705882353 + }, + { + "repo": "walles/px", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "caplog.text", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "get_users_at(lastline, now, testtime)", + "got": "get_users_at(lastline, now, testtime)", + "got_raw": "get_users_at(lastline, now, testtime)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "get_users_at(lastline, now, datetime.datetime.now(TIMEZONE))", + "got": "get_users_at( lastline, now, datetime.datetime(2016, 4, 3, 11, 55, tzinfo=TIMEZONE) )", + "got_raw": "get_users_at( lastline, now, datetime.datetime(2016, 4, 3, 11, 55, tzinfo=TIMEZONE) )", + "exact_match": false, + "code_bleu": 0.4055105980173464, + "edit_similarity": 0.7724137931034483 + }, + { + "repo": "walles/px", + "expected": "datetime.timedelta( 0, hours=1, minutes=29 )", + "got": "datetime.timedelta(1, 1700)", + "got_raw": "datetime.timedelta(1, 1700)", + "exact_match": false, + "code_bleu": 0.3032978383469967, + "edit_similarity": 0.647887323943662 + }, + { + "repo": "walles/px", + "expected": "datetime.timedelta( 4, hours=1, minutes=29 )", + "got": "datetime.timedelta( 0, hours=5, minutes=29", + "got_raw": "datetime.timedelta( 0, hours=5, minutes=29", + "exact_match": false, + "code_bleu": 0.49525406853956827, + "edit_similarity": 0.9302325581395349 + }, + { + "repo": "walles/px", + "expected": "datetime.timedelta( 34, hours=1, minutes=29 )", + "got": "datetime.timedelta(34)", + "got_raw": "datetime.timedelta(34)", + "exact_match": false, + "code_bleu": 0.3603841779658777, + "edit_similarity": 0.6567164179104478 + }, + { + "repo": "walles/px", + "expected": "get_users_at( lastline, now, datetime.datetime.now(TIMEZONE) )", + "got": "get_users_at(lastline, now)", + "got_raw": "get_users_at(lastline, now)", + "exact_match": false, + "code_bleu": 0.33635856610148585, + "edit_similarity": 0.6067415730337079 + }, + { + "repo": "walles/px", + "expected": "get_users_at( lastline, now, datetime.datetime(2016, 4, 1, 11, 8, tzinfo=TIMEZONE), )", + "got": "get_users_at(lastline, now)", + "got_raw": "get_users_at(lastline, now)", + "exact_match": false, + "code_bleu": 0.1940530189047034, + "edit_similarity": 0.48214285714285715 + }, + { + "repo": "walles/px", + "expected": "get_users_at( lastline, now, datetime.datetime(2016, 4, 3, 11, 54, tzinfo=TIMEZONE), )", + "got": "get_users_at(lastline, now, now)", + "got_raw": "get_users_at(lastline, now, now)", + "exact_match": false, + "code_bleu": 0.23096118576933522, + "edit_similarity": 0.5254237288135594 + }, + { + "repo": "walles/px", + "expected": "get_users_at( lastline, now, datetime.datetime(2016, 3, 9, 13, 26, tzinfo=TIMEZONE), )", + "got": "get_users_at(lastline, now)", + "got_raw": "get_users_at(lastline, now)", + "exact_match": false, + "code_bleu": 0.1940530189047034, + "edit_similarity": 0.4778761061946903 + }, + { + "repo": "walles/px", + "expected": "get_users_at( lastline, now, datetime.datetime(2016, 3, 26, 22, 4, tzinfo=TIMEZONE), )", + "got": "get_users_at(lastline, now, now)", + "got_raw": "get_users_at(lastline, now, now)", + "exact_match": false, + "code_bleu": 0.23096118576933522, + "edit_similarity": 0.5254237288135594 + }, + { + "repo": "walles/px", + "expected": "get_users_at( lastline, now, datetime.datetime(2016, 3, 26, 22, 9, tzinfo=TIMEZONE), )", + "got": "get_users_at(lastline, now)", + "got_raw": "get_users_at(lastline, now)", + "exact_match": false, + "code_bleu": 0.1940530189047034, + "edit_similarity": 0.4778761061946903 + }, + { + "repo": "walles/px", + "expected": "get_users_at( lastline, now, datetime.datetime(2016, 4, 4, 23, 10, tzinfo=TIMEZONE), )", + "got": "get_users_at( lastline, now, now )", + "got_raw": "get_users_at( lastline, now, now )", + "exact_match": false, + "code_bleu": 0.23096118576933522, + "edit_similarity": 0.55 + }, + { + "repo": "walles/px", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "1921988 * 1024", + "got": "1073741824", + "got_raw": "1073741824", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "walles/px", + "expected": "371474308 * 1024", + "got": "1073741824", + "got_raw": "1073741824", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "walles/px", + "expected": "total_bytes - available_kb * 1024", + "got": "available_kb * 1024", + "got_raw": "available_kb * 1024", + "exact_match": false, + "code_bleu": 0.5045378491522287, + "edit_similarity": 0.7307692307692307 + }, + { + "repo": "walles/px", + "expected": "7", + "got": "7", + "got_raw": "7", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "4", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "p1", + "got": "p1", + "got_raw": "p1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "p2", + "got": "p2", + "got_raw": "p2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "123", + "got": "123", + "got_raw": "123", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "now", + "got": "now", + "got_raw": "now", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "1234", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "walles/px", + "expected": "\"0%\"", + "got": "\"0.03%\"", + "got_raw": "\"0.03%\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "walles/px", + "expected": "\"6%\"", + "got": "1.97", + "got_raw": "1.97", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "pids", + "got": "pids", + "got_raw": "pids", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "0.03", + "got": "datetime.timedelta(0", + "got_raw": "datetime.timedelta(0, 3)", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "walles/px", + "expected": "8", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "close", + "got": "close", + "got_raw": "close", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "px_processinfo.to_relative_start_string(base, close)", + "got": "\"Mon Mar 7 09:33:11 2016\"", + "got_raw": "\"Mon Mar 7 09:33:11 2016\"", + "exact_match": false, + "code_bleu": 0.0625, + "edit_similarity": 0.1282051282051282 + }, + { + "repo": "walles/px", + "expected": "[ (\"\" + str(parent_proc), parent_proc), (\" \" + str(child_proc), child_proc), ]", + "got": "[ (\"parent\", parent_proc), (\"child\", child_proc) ]", + "got_raw": "[ (\"parent\", parent_proc), (\"child\", child_proc) ]", + "exact_match": false, + "code_bleu": 0.26801642121457603, + "edit_similarity": 0.75 + }, + { + "repo": "walles/px", + "expected": "[ px_terminal.bold(\"bar(47536)\") + \": [PIPE] ->0xAda\", px_terminal.bold(\"foo(47536)\") + \": [PIPE] ->0xAda\", ]", + "got": "[ \"foo 47536 0xAda 0xE0e 25\", \"bar 47536 0xAda 0xE0e 25\", ]", + "got_raw": "[ \"foo 47536 0xAda 0xE0e 25\", \"bar 47536 0xAda 0xE0e 25\", ]", + "exact_match": false, + "code_bleu": 0.09909492954041421, + "edit_similarity": 0.39285714285714285 + }, + { + "repo": "walles/px", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "\"q\"", + "got": "\"q\"", + "got_raw": "\"q\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "\"123\"", + "got": "\"123\"", + "got_raw": "\"123\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "\"1234\"", + "got": "\"123\"", + "got_raw": "\"123\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "walles/px", + "expected": "\"12345\"", + "got": "\"12345\"", + "got_raw": "\"12345\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "\"12345 \"", + "got": "\"12345\"", + "got_raw": "\"12345\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.875 + }, + { + "repo": "walles/px", + "expected": "mid_bold + \" \"", + "got": "\"12345\"", + "got_raw": "\"12345\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "walles/px", + "expected": "[\"ab\", \"\\x1b[1m\", \"c\", \"\\x1b[22m\", \"de\"]", + "got": "[\"ab\", \"bold(c)\", \"de\"]", + "got_raw": "[\"ab\", \"bold(c)\", \"de\"]", + "exact_match": false, + "code_bleu": 0.3181818181818182, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "walles/px", + "expected": "[ r\" PID COMMAND USERNAME CPU CPUTIME RAM COMMANDLINE\", r\"47536 \ud83d\ude00 root 0% 0.03s 0% /usr/bin/\ud83d\ude00\", ]", + "got": "[[\"/usr/bin/\ud83d\ude00\"]]", + "got_raw": "[[\"/usr/bin/\ud83d\ude00\"]]", + "exact_match": false, + "code_bleu": 0.188835962334798, + "edit_similarity": 0.24778761061946902 + }, + { + "repo": "walles/px", + "expected": "[ \"\\x1b[1m PID COMMAND USERNAME CPU CPUTIME RAM COMMANDLINE\\x1b[22m\", \"47536 fluff \" + px_terminal.faint(\"root\") + \" \" + px_terminal.faint(\" 0%\") + \" \" + px_terminal.bold(\" 0.03s\") + \" \" + px_terminal.faint(\" 0%\") + \" /usr/bin/fluff 1234\", ]", + "got": "[[\"1234\"]]", + "got_raw": "[[\"1234\"]]", + "exact_match": false, + "code_bleu": 0.035968754730437716, + "edit_similarity": 0.06374501992031872 + }, + { + "repo": "walles/px", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "\"--\"", + "got": "0.0", + "got_raw": "0.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "SCREEN_ROWS", + "got": "SCREEN_ROWS", + "got_raw": "SCREEN_ROWS", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "px_top.CMD_QUIT", + "got": "\"q\"", + "got_raw": "\"q\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "px_top.CMD_RESIZE", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "lines[0].replace(CSI, \"CSI\")", + "got": "lines[0]", + "got_raw": "lines[0]", + "exact_match": false, + "code_bleu": 0.36363636363636365, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "walles/px", + "expected": "lines[-1].replace(CSI, \"CSI\")", + "got": "lines[-1]", + "got_raw": "lines[-1]", + "exact_match": false, + "code_bleu": 0.4166666666666667, + "edit_similarity": 0.47368421052631576 + }, + { + "repo": "walles/px", + "expected": "[]", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "[\"parent(1)\", \" child(2)\"]", + "got": "[px_process.PxProcess(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33", + "got_raw": "[px_process.PxProcess(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33", + "exact_match": false, + "code_bleu": 0.018938767636553357, + "edit_similarity": 0.10588235294117647 + }, + { + "repo": "walles/px", + "expected": "[f\"Lumpur... ({px_terminal.bold('3\u00d7')})\"]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.09302325581395349 + }, + { + "repo": "walles/px", + "expected": "[\"root(1)\", \" find-me(2)\", \" uninteresting(3)\"]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.16988673071467442, + "edit_similarity": 0.0784313725490196 + }, + { + "repo": "walles/px", + "expected": "[\"root(1)\", f\" {px_terminal.bold('find-me')}(2)\"]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.07692307692307693 + }, + { + "repo": "walles/px", + "expected": "[\"root(1)\", f\" process... ({px_terminal.bold('2\u00d7')})\"]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.07017543859649122 + }, + { + "repo": "walles/px", + "expected": "[ \"Kuala(1)\" ]", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "walles/px", + "expected": "[ \"root(1)\", f\" {px_terminal.bold('process')}(2)\", f\" {px_terminal.bold('process')}(3)\", ]", + "got": "[px_process.PxProcess(1, \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"", + "got_raw": "[px_process.PxProcess(1, \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"", + "exact_match": false, + "code_bleu": 0.017490954988811324, + "edit_similarity": 0.102803738317757 + }, + { + "repo": "walles/px", + "expected": "[ \"root(1)\", f\" process... ({px_terminal.bold('4\u00d7')})\", \" process(4)\", \" subprocess(7)\", ]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.1081097377275201, + "edit_similarity": 0.043478260869565216 + }, + { + "repo": "walles/px", + "expected": "(\"0KB\", \"7KB\")", + "got": "(\"0B\", \"7.0KB\")", + "got_raw": "(\"0B\", \"7.0KB\")", + "exact_match": false, + "code_bleu": 0.32466791547509893, + "edit_similarity": 0.896551724137931 + }, + { + "repo": "walles/px", + "expected": "(\"0MB\", \"7MB\")", + "got": "(\"0MB\", \"7MB\")", + "got_raw": "(\"0MB\", \"7MB\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "(\"0GB\", \"7GB\")", + "got": "(\"0GB\", \"7GB\")", + "got_raw": "(\"0GB\", \"7GB\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "(\"0TB\", \"7TB\")", + "got": "(\"0TB\", \"7TB\")", + "got_raw": "(\"0TB\", \"7TB\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "(\"0B\", \"7167B\")", + "got": "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0", + "got_raw": "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0", + "exact_match": false, + "code_bleu": 0.01391004452078366, + "edit_similarity": 0.04195804195804196 + }, + { + "repo": "walles/px", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "1235", + "got": "\"1235\"", + "got_raw": "\"1235\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8 + }, + { + "repo": "walles/px", + "expected": "args)", + "got": "args)", + "got_raw": "args)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "\"kalas\"", + "got": "\"kalas\"", + "got_raw": "\"kalas\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "walles/px", + "expected": "px.__doc__)", + "got": "\"Usage:", + "got_raw": "\"Usage: px [OPTIONS] COMMAND [ARGS]\")", + "exact_match": false, + "code_bleu": 0.2849383821193472, + "edit_similarity": 0.0 + }, + { + "repo": "walles/px", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "url)", + "got": "url)", + "got_raw": "url)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "res)", + "got": "res[\"media_id\"])", + "got_raw": "res[\"media_id\"])", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.4 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "group)", + "got": "group[\"count\"])", + "got_raw": "group[\"count\"])", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "users)", + "got": "users)", + "got_raw": "users)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "result)", + "got": "result)", + "got_raw": "result)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "ticket)", + "got": "\"wx_card\"", + "got_raw": "\"wx_card\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.125 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "client2)", + "got": "client2)", + "got_raw": "client2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "card_ext)", + "got": "card_ext)", + "got_raw": "card_ext)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "len(res))", + "got": "len(res))", + "got_raw": "len(res))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "group_id)", + "got": "group_id)", + "got_raw": "group_id)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "authorize_url)", + "got": "\"authorize\"", + "got_raw": "\"authorize\" in authorize_url", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.72 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "res[\"openid\"])", + "got": "res[\"openid\"])", + "got_raw": "res[\"openid\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "result[\"errcode\"])", + "got": "result[\"errcode\"])", + "got_raw": "result[\"errcode\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "token[\"expires_in\"])", + "got": "token[\"expires_in\"])", + "got_raw": "token[\"expires_in\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "res[\"access_token\"])", + "got": "res[\"access_token\"])", + "got_raw": "res[\"access_token\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "result[\"expires_in\"])", + "got": "result[\"expires_in\"])", + "got_raw": "result[\"expires_in\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "result[\"option_name\"])", + "got": "result[\"option\"])", + "got_raw": "result[\"option\"])", + "exact_match": false, + "code_bleu": 0.45180100180492244, + "edit_similarity": 0.8717948717948718 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "result[\"option_value\"])", + "got": "result[\"option_value\"])", + "got_raw": "result[\"option_value\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "e.response.status_code)", + "got": "e.error_code)", + "got_raw": "e.error_code)", + "exact_match": false, + "code_bleu": 0.3387551654364098, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "result[\"pre_auth_code\"])", + "got": "result[\"pre_auth_code\"])", + "got_raw": "result[\"pre_auth_code\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "self.client.access_token)", + "got": "token[\"component_access_token\"])", + "got_raw": "token[\"component_access_token\"])", + "exact_match": false, + "code_bleu": 0.22751007189242223, + "edit_similarity": 0.631578947368421 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "token[\"component_access_token\"])", + "got": "token[\"access_token\"])", + "got_raw": "token[\"access_token\"])", + "exact_match": false, + "code_bleu": 0.45180100180492244, + "edit_similarity": 0.8148148148148148 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "result[\"authorizer_access_token\"])", + "got": "result[\"authorizer_access_token\"])", + "got_raw": "result[\"authorizer_access_token\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "result[\"authorizer_refresh_token\"])", + "got": "result[\"authorizer_refresh_token\"])", + "got_raw": "result[\"authorizer_refresh_token\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "result[\"authorizer_info\"][\"alias\"])", + "got": "result[\"authorizer_info\"][\"authorizer_name\"])", + "got_raw": "result[\"authorizer_info\"][\"authorizer_name\"])", + "exact_match": false, + "code_bleu": 0.6606328636027614, + "edit_similarity": 0.825 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "inspect.ismethod(self.client.fetch_access_token))", + "got": "callable(self.client.fetch_access_token))", + "got_raw": "callable(self.client.fetch_access_token))", + "exact_match": false, + "code_bleu": 0.7200914210127765, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "result[\"authorization_info\"][\"authorizer_appid\"])", + "got": "result[\"authorizer_appid\"])", + "got_raw": "result[\"authorizer_appid\"])", + "exact_match": false, + "code_bleu": 0.5255602595335715, + "edit_similarity": 0.7105263157894737 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "WeChatErrorCode(-1))", + "got": "WeChatErrorCode.SYSTEM_BUSY)", + "got_raw": "WeChatErrorCode.SYSTEM_BUSY)", + "exact_match": false, + "code_bleu": 0.26509024292235017, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "WeChatErrorCode.SYSTEM_ERROR.value)", + "got": "WeChatErrorCode.UNKNOWN_ERROR)", + "got_raw": "WeChatErrorCode.UNKNOWN_ERROR)", + "exact_match": false, + "code_bleu": 0.3387551654364098, + "edit_similarity": 0.7076923076923077 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "WeChatErrorCode.UNAUTHORIZED_API.value)", + "got": "WeChatErrorCode.INVALID_ACCESS_TOKEN.value)", + "got_raw": "WeChatErrorCode.INVALID_ACCESS_TOKEN.value)", + "exact_match": false, + "code_bleu": 0.488923022434901, + "edit_similarity": 0.6585365853658537 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "WeChatErrorCode.EXPIRED_ACCESS_TOKEN.value)", + "got": "WeChatErrorCode.INVALID_ACCESS_TOKEN.value)", + "got_raw": "WeChatErrorCode.INVALID_ACCESS_TOKEN.value)", + "exact_match": false, + "code_bleu": 0.488923022434901, + "edit_similarity": 0.8837209302325582 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply)", + "got": "reply)", + "got_raw": "reply)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply.type)", + "got": "reply.type)", + "got_raw": "reply.type)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply.target)", + "got": "reply.from_user)", + "got_raw": "reply.from_user)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6206896551724138 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply.source)", + "got": "reply.sender)", + "got_raw": "reply.sender)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6923076923076923 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply.content)", + "got": "reply.content)", + "got_raw": "reply.content)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "AttributeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "isinstance(reply, str))", + "got": "reply.render)", + "got_raw": "reply.render)", + "exact_match": false, + "code_bleu": 0.2272202082191573, + "edit_similarity": 0.3888888888888889 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "isinstance(reply, EmptyReply))", + "got": "isinstance(reply, EmptyReply))", + "got_raw": "isinstance(reply, EmptyReply))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "encrypted)", + "got": "encrypted)", + "got_raw": "encrypted)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "data[\"refund_id\"])", + "got": "data[\"refund_id\"])", + "got_raw": "data[\"refund_id\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg_dict[\"Content\"])", + "got": "msg_dict[\"ToUserName\"])", + "got_raw": "msg_dict[\"ToUserName\"])", + "exact_match": false, + "code_bleu": 0.45180100180492244, + "edit_similarity": 0.6976744186046512 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "data[\"out_trade_no\"])", + "got": "data[\"return_code\"])", + "got_raw": "data[\"return_code\"])", + "exact_match": false, + "code_bleu": 0.45180100180492244, + "edit_similarity": 0.5853658536585366 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "data[\"out_refund_no\"])", + "got": "data[\"appid\"])", + "got_raw": "data[\"appid\"])", + "exact_match": false, + "code_bleu": 0.45180100180492244, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "data[\"transaction_id\"])", + "got": "data[\"return_code\"])", + "got_raw": "data[\"return_code\"])", + "exact_match": false, + "code_bleu": 0.45180100180492244, + "edit_similarity": 0.6511627906976745 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg_dict[\"FromUserName\"])", + "got": "msg_dict[\"MsgType\"])", + "got_raw": "msg_dict[\"MsgType\"])", + "exact_match": false, + "code_bleu": 0.45180100180492244, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "InvalidSignatureException)", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "decrypted[\"watermark\"][\"appid\"])", + "got": "decrypted[\"appid\"])", + "got_raw": "decrypted[\"appid\"])", + "exact_match": false, + "code_bleu": 0.5255602595335715, + "edit_similarity": 0.7450980392156863 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "xml2)", + "got": "xml2)", + "got_raw": "xml2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "response)", + "got": "response)", + "got_raw": "response)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "event.id)", + "got": "event.msg_id)", + "got_raw": "event.msg_id)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8181818181818182 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "event.key)", + "got": "event.event_key)", + "got_raw": "event.event_key)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "event.url)", + "got": "event.event_key)", + "got_raw": "event.event_key)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.5384615384615384 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "\"by time\")", + "got": "\"by time\")", + "got_raw": "\"by time\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "event.count)", + "got": "event.send_pics_info.count)", + "got_raw": "event.send_pics_info.count)", + "exact_match": false, + "code_bleu": 0.46713797772820015, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "event.scale)", + "got": "event.scale)", + "got_raw": "event.scale)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "event.label)", + "got": "", + "got_raw": "", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "event.bssid)", + "got": "event.device_no)", + "got_raw": "event.device_no)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "event.source)", + "got": "event.source)", + "got_raw": "event.source)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "event.target)", + "got": "event.target)", + "got_raw": "event.target)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "event.ticket)", + "got": "event.ticket)", + "got_raw": "event.ticket)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "res[\"errcode\"])", + "got": "res[\"errcode\"])", + "got_raw": "res[\"errcode\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "len(external_contact_list))", + "got": "len(external_contact_list))", + "got_raw": "len(external_contact_list))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "xml)", + "got": "xml)", + "got_raw": "xml)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "field.to_xml(value))", + "got": "field.to_xml(value))", + "got_raw": "field.to_xml(value))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "field.to_xml(content))", + "got": "field.to_xml(content))", + "got_raw": "field.to_xml(content))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "field.to_xml(articles))", + "got": "field.to_xml(articles))", + "got_raw": "field.to_xml(articles))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "client)", + "got": "client.access_token)", + "got_raw": "client.access_token)", + "exact_match": false, + "code_bleu": 0.3976353643835253, + "edit_similarity": 0.5185185185185185 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "res[\"errcode\"])", + "got": "res[\"errcode\"])", + "got_raw": "res[\"errcode\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "token[\"expires_in\"])", + "got": "token[\"expires_in\"])", + "got_raw": "token[\"expires_in\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "token[\"access_token\"])", + "got": "token[\"access_token\"])", + "got_raw": "token[\"access_token\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "self.client.access_token)", + "got": "token[\"refresh_token\"])", + "got_raw": "token[\"refresh_token\"])", + "exact_match": false, + "code_bleu": 0.22751007189242223, + "edit_similarity": 0.4583333333333333 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.id)", + "got": "msg.msg_id)", + "got_raw": "msg.msg_id)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.url)", + "got": "msg.url)", + "got_raw": "msg.url)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.image)", + "got": "msg.pic_url)", + "got_raw": "msg.pic_url)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.scale)", + "got": "msg.scale)", + "got_raw": "msg.scale)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.label)", + "got": "msg.label)", + "got_raw": "msg.label)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.title)", + "got": "msg.title)", + "got_raw": "msg.title)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.source)", + "got": "msg.from_user)", + "got_raw": "msg.from_user)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.56 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.target)", + "got": "msg.target)", + "got_raw": "msg.target)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.format)", + "got": "msg.format)", + "got_raw": "msg.format)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.app_id)", + "got": "msg.appid)", + "got_raw": "msg.appid)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.9523809523809523 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.content)", + "got": "msg.content)", + "got_raw": "msg.content)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.media_id)", + "got": "msg.media_id)", + "got_raw": "msg.media_id)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.page_path)", + "got": "msg.page_path)", + "got_raw": "msg.page_path)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.thumb_url)", + "got": "msg.thumb_url)", + "got_raw": "msg.thumb_url)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.location_x)", + "got": "msg.location_x)", + "got_raw": "msg.location_x)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.location_y)", + "got": "msg.location_y)", + "got_raw": "msg.location_y)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.recognition)", + "got": "msg.recognition)", + "got_raw": "msg.recognition)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.description)", + "got": "msg.description)", + "got_raw": "msg.description)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "url)", + "got": "\"wechat-oauth-qrconnect\")", + "got_raw": "\"wechat-oauth-qrconnect\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.20689655172413793 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "res)", + "got": "res)", + "got_raw": "res)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "authorize_url)", + "got": "\"authorize\"", + "got_raw": "\"authorize\" in authorize_url", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.72 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "res[\"openid\"])", + "got": "res[\"openid\"])", + "got_raw": "res[\"openid\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "res[\"access_token\"])", + "got": "res[\"access_token\"])", + "got_raw": "res[\"access_token\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "e.response.status_code)", + "got": "e.status_code)", + "got_raw": "e.status_code)", + "exact_match": false, + "code_bleu": 0.5065571237677283, + "edit_similarity": 0.7567567567567568 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.key)", + "got": "msg.event_key)", + "got_raw": "msg.event_key)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.url)", + "got": "msg.event_key)", + "got_raw": "msg.event_key)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.45454545454545453 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.type)", + "got": "msg[\"msg_type\"])", + "got_raw": "msg[\"msg_type\"])", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.64 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.code)", + "got": "msg.code)", + "got_raw": "msg.code)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "xml)", + "got": "xml)", + "got_raw": "xml)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "sign)", + "got": "sign)", + "got_raw": "sign)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "sign)", + "got": "sign)", + "got_raw": "sign)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "response)", + "got": "response)", + "got_raw": "response)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "[{\"encrypt_certificate\": \"\", \"serial_no\": \"\"}, {\"encrypt_certificate\": \"\", \"serial_no\": \"\"}])", + "got": "client.update_certificates()", + "got_raw": "client.update_certificates()", + "exact_match": false, + "code_bleu": 0.06204820142520606, + "edit_similarity": 0.2809917355371901 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "response)", + "got": "response)", + "got_raw": "response)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "\"SUCCESS\")", + "got": "\"SUCCESS\")", + "got_raw": "\"SUCCESS\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "response[\"data\"])", + "got": "response)", + "got_raw": "response)", + "exact_match": false, + "code_bleu": 0.23784142300054423, + "edit_similarity": 0.6923076923076923 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply.image)", + "got": "reply.image)", + "got_raw": "reply.image)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply.voice)", + "got": "reply.voice)", + "got_raw": "reply.voice)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply.title)", + "got": "reply.title)", + "got_raw": "reply.title)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply.source)", + "got": "reply.source)", + "got_raw": "reply.source)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply.target)", + "got": "reply.target)", + "got_raw": "reply.target)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply.media_id)", + "got": "reply[\"image\"])", + "got_raw": "reply[\"image\"])", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply.render())", + "got": "reply.content)", + "got_raw": "reply.content)", + "exact_match": false, + "code_bleu": 0.3387551654364098, + "edit_similarity": 0.6206896551724138 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply.music_url)", + "got": "reply.music_url)", + "got_raw": "reply.music_url)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "create_time in r)", + "got": "create_time in r)", + "got_raw": "create_time in r)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "url)", + "got": "url)", + "got_raw": "url)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "token[\"expires_in\"])", + "got": "token[\"expires_in\"])", + "got_raw": "token[\"expires_in\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "client.access_token)", + "got": "client.access_token)", + "got_raw": "client.access_token)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "token[\"access_token\"])", + "got": "token[\"access_token\"])", + "got_raw": "token[\"access_token\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "isinstance(client.session, RedisStorage))", + "got": "session)", + "got_raw": "session)", + "exact_match": false, + "code_bleu": 0.1321341238891912, + "edit_similarity": 0.32653061224489793 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "isinstance(client.session, MemoryStorage))", + "got": "client._storage)", + "got_raw": "client._storage)", + "exact_match": false, + "code_bleu": 0.22583677695760654, + "edit_similarity": 0.5172413793103449 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "isinstance(client.session, MemcachedStorage))", + "got": "session)", + "got_raw": "session)", + "exact_match": false, + "code_bleu": 0.1321341238891912, + "edit_similarity": 0.3018867924528302 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "obj.xxx)", + "got": "obj.xxx)", + "got_raw": "obj.xxx)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "signature)", + "got": "signature)", + "got_raw": "signature)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "obj.xxx is None)", + "got": "obj)", + "got_raw": "obj)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.4 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "InvalidSignatureException)", + "got": "False)", + "got_raw": "False)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.25 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "target_string.encode(\"utf-8\"))", + "got": "b64_encode=True)", + "got_raw": "b64_encode=True)", + "exact_match": false, + "code_bleu": 0.20531663540747672, + "edit_similarity": 0.391304347826087 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "res)", + "got": "res[\"ip_list\"])", + "got_raw": "res[\"ip_list\"])", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "100)", + "got": "{", + "got_raw": "{ \"active_user_count\": 1, \"active_user_list\": [ { \"userid\": \"202003270000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "dict)", + "got": "dict)", + "got_raw": "dict)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "client)", + "got": "client.access_token)", + "got_raw": "client.access_token)", + "exact_match": false, + "code_bleu": 0.3976353643835253, + "edit_similarity": 0.5185185185185185 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "user_id)", + "got": "user_id)", + "got_raw": "user_id)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "len(res))", + "got": "len(res))", + "got_raw": "len(res))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "signature)", + "got": "signature)", + "got_raw": "signature)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "res[\"id\"])", + "got": "res[\"id\"])", + "got_raw": "res[\"id\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply)", + "got": "reply)", + "got_raw": "reply)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply.type)", + "got": "reply[\"msg_type\"])", + "got_raw": "reply[\"msg_type\"])", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.6896551724137931 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply.agent)", + "got": "reply.media_id)", + "got_raw": "reply.media_id)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.5925925925925926 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply.target)", + "got": "reply.from_user)", + "got_raw": "reply.from_user)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6206896551724138 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply.source)", + "got": "reply.from_user)", + "got_raw": "reply.from_user)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6206896551724138 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply.content)", + "got": "reply.content)", + "got_raw": "reply.content)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "reply is None)", + "got": "reply is None)", + "got_raw": "reply is None)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "AttributeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "isinstance(reply, str))", + "got": "reply)", + "got_raw": "reply)", + "exact_match": false, + "code_bleu": 0.16988673071467442, + "edit_similarity": 0.41379310344827586 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.key)", + "got": "msg.event_key)", + "got_raw": "msg.event_key)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.url)", + "got": "msg.event_key)", + "got_raw": "msg.event_key)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.45454545454545453 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.type)", + "got": "msg.msg_type)", + "got_raw": "msg.msg_type)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8181818181818182 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.agent)", + "got": "msg.agent_id)", + "got_raw": "msg.agent_id)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8695652173913043 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.event)", + "got": "msg.event)", + "got_raw": "msg.event)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wechatpy/wechatpy", + "expected": "msg.job_id)", + "got": "msg.job_id)", + "got_raw": "msg.job_id)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "ref2.ref", + "got": "ref2.ref", + "got_raw": "ref2.ref", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.NO_CONTENT", + "got_raw": "HTTPStatus.NO_CONTENT", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "refresh['jti']", + "got": "refresh['jti']", + "got_raw": "refresh['jti']", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "ContentType.xml", + "got": "str(ContentType.xml)", + "got_raw": "str(ContentType.xml)", + "exact_match": false, + "code_bleu": 0.4111336169005197, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "request.headers", + "got": "request.headers", + "got_raw": "request.headers", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "dict[str, int])", + "got": "Path)", + "got_raw": "Path)", + "exact_match": false, + "code_bleu": 0.153509990260506, + "edit_similarity": 0.2 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "dict[str, int])", + "got": "Body[str])", + "got_raw": "Body[str])", + "exact_match": false, + "code_bleu": 0.3227150012892303, + "edit_similarity": 0.56 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "dict[str, str])", + "got": "dict[str, str])", + "got_raw": "dict[str, str])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "ContentType.json", + "got": "str(ContentType.json)", + "got_raw": "str(ContentType.json)", + "exact_match": false, + "code_bleu": 0.4111336169005197, + "edit_similarity": 0.8648648648648649 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.FOUND", + "got": "HTTPStatus.FOUND", + "got_raw": "HTTPStatus.FOUND", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.CREATED", + "got_raw": "HTTPStatus.CREATED", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.CREATED", + "got_raw": "HTTPStatus.CREATED", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "self.request.FILES", + "got": "self.parsed_file_metadata.model_fields", + "got_raw": "self.parsed_file_metadata.model_fields", + "exact_match": false, + "code_bleu": 0.3860973950960897, + "edit_similarity": 0.32142857142857145 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "OpenAPIFormat.IPV4", + "got": "'ipv4'", + "got_raw": "'ipv4'", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.NO_CONTENT", + "got_raw": "HTTPStatus.NO_CONTENT", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.717948717948718 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CREATED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CONFLICT", + "got": "HTTPStatus.BAD_REQUEST", + "got_raw": "HTTPStatus.BAD_REQUEST", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.5853658536585366 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.ACCEPTED", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6875 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CONFLICT", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.75 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "OpenAPIType.INTEGER", + "got": "OpenAPIType.INTEGER", + "got_raw": "OpenAPIType.INTEGER", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.CONFLICT", + "got": "HTTPStatus.CONFLICT", + "got_raw": "HTTPStatus.CONFLICT", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.FORBIDDEN", + "got": "HTTPStatus.FORBIDDEN", + "got_raw": "HTTPStatus.FORBIDDEN", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "OpenAPIFormat.BINARY", + "got": "\"binary\"", + "got_raw": "\"binary\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "HTTPStatus.FORBIDDEN", + "got": "HTTPStatus.NOT_ACCEPTABLE", + "got_raw": "HTTPStatus.NOT_ACCEPTABLE", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "{}", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "[]", + "got": "{'hello':", + "got_raw": "{'hello': 'world'}", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "()", + "got": "expected_capture", + "got_raw": "expected_capture", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "['a']", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.30120066786994826, + "edit_similarity": 0.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "[1, 2]", + "got": "snapshot({ 'cookies': [ NewCookie(name='foo', value='bar') ] })", + "got_raw": "snapshot({ 'cookies': [ NewCookie(name='foo', value='bar') ] })", + "exact_match": false, + "code_bleu": 0.07658412276041004, + "edit_similarity": 0.11594202898550725 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "[1, 2]", + "got": "{'status_code': 200}", + "got_raw": "{'status_code': 200}", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "[1, 2]", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.18072040072196896, + "edit_similarity": 0.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "['a', 'b']", + "got": "headers", + "got_raw": "headers", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "{'detail'}", + "got": "{'error'}", + "got_raw": "{'error'}", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.5263157894736842 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "('', (), {})", + "got": "{'': None}", + "got_raw": "{'': None}", + "exact_match": false, + "code_bleu": 0.18037106415283274, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "(_BodyModel,)", + "got": "_BodyModel", + "got_raw": "_BodyModel", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.8695652173913043 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "['card', 'big']", + "got": "snapshot({ 'payment_method_id': 'card', 'payment_amount': 'big', })", + "got_raw": "snapshot({ 'payment_method_id': 'card', 'payment_amount': 'big', })", + "exact_match": false, + "code_bleu": 0.1350862565735141, + "edit_similarity": 0.3170731707317073 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "{HTTPStatus.OK}", + "got": "{'200'", + "got_raw": "{'200', '400', '401'}", + "exact_match": false, + "code_bleu": 0.2149139863647084, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "{'user': 'name'}", + "got": "{ 'foo': 'bar' }", + "got_raw": "{ 'foo': 'bar' }", + "exact_match": false, + "code_bleu": 0.32466791547509893, + "edit_similarity": 0.5625 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "{'key': 'value'}", + "got": "request_data['extra']", + "got_raw": "request_data['extra']", + "exact_match": false, + "code_bleu": 0.24171003178854145, + "edit_similarity": 0.21621621621621623 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "{'result': 'done'}", + "got": "{'status': 'accepted'}", + "got_raw": "{'status': 'accepted'}", + "exact_match": false, + "code_bleu": 0.32466791547509893, + "edit_similarity": 0.55 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "[{'noScheme': []}]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.1767766952966369, + "edit_similarity": 0.2 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "['first', 'second']", + "got": "list(aiterator)", + "got_raw": "list(aiterator)", + "exact_match": false, + "code_bleu": 0.24171003178854145, + "edit_similarity": 0.29411764705882354 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "[{'testScheme': []}]", + "got": "[SecurityRequirement(schemes=[_WithScheme()])]", + "got_raw": "[SecurityRequirement(schemes=[_WithScheme()])]", + "exact_match": false, + "code_bleu": 0.13065113298388567, + "edit_similarity": 0.3939393939393939 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "{'username': 'admin'}", + "got": "{ 'raw_data': 'raw_data'", + "got_raw": "{ 'raw_data': 'raw_data', }", + "exact_match": false, + "code_bleu": 0.3181082915068203, + "edit_similarity": 0.4888888888888889 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "{'message': 'Success'}", + "got": "snapshot({ 'custom_header': 'custom_value' })", + "got_raw": "snapshot({ 'custom_header': 'custom_value' })", + "exact_match": false, + "code_bleu": 0.19070828081828378, + "edit_similarity": 0.3283582089552239 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "{'X-API-Token': '123'}", + "got": "snapshot({})", + "got_raw": "snapshot({})", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "{'full_name': 'Example'}", + "got": "{'forward_ref': 'ForwardRef'}", + "got_raw": "{'forward_ref': 'ForwardRef'}", + "exact_match": false, + "code_bleu": 0.32466791547509893, + "edit_similarity": 0.49056603773584906 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "{'session_id': 'unique'}", + "got": "{'user_id': '1'}", + "got_raw": "{'user_id': '1'}", + "exact_match": false, + "code_bleu": 0.32466791547509893, + "edit_similarity": 0.65 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "{'user_id': str(user.pk)}", + "got": "snapshot({ 'username': user.username })", + "got_raw": "snapshot({ 'username': user.username })", + "exact_match": false, + "code_bleu": 0.21200626759025185, + "edit_similarity": 0.46875 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "[{'name': _EXAMPLE_NAME}]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.16988673071467442, + "edit_similarity": 0.14814814814814814 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "{'query': expected_query_value}", + "got": "{'query': expected_query_value}", + "got_raw": "{'query': expected_query_value}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "{'username': 'validated-admin'}", + "got": "snapshot({ 'foo': 'bar' })", + "got_raw": "snapshot({ 'foo': 'bar' })", + "exact_match": false, + "code_bleu": 0.19070828081828378, + "edit_similarity": 0.3508771929824561 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "str", + "got": "str", + "got_raw": "str", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "int", + "got": "int", + "got_raw": "int", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "iss", + "got": "iss", + "got_raw": "iss", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "bool", + "got": "int", + "got_raw": "int", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "props", + "got": "props", + "got_raw": "props", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "header", + "got": "header", + "got_raw": "header", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "regular", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "headers", + "got": "headers", + "got_raw": "headers", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "expected", + "got": "snapshot({", + "got_raw": "snapshot({ 'error': 'Endpoint', })", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "username", + "got": "snapshot({", + "got_raw": "snapshot({ 'username': username })", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "_MAXIMUM", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "expected", + "got": "{'error':", + "got_raw": "{'error': expected}", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "audience", + "got": "audience", + "got_raw": "audience", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "last_name", + "got": "{'query':", + "got_raw": "{'query': last_name}", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.1111111111111111 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "field_map", + "got": "field_map", + "got_raw": "field_map", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "request_id", + "got": "request_id", + "got_raw": "request_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "TypeError)", + "got": "UnsolvableAnnotationsError)", + "got_raw": "UnsolvableAnnotationsError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3783783783783784 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "controller", + "got": "controller", + "got_raw": "controller", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "status_code", + "got": "status_code", + "got_raw": "status_code", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "status_code", + "got": "status_code", + "got_raw": "status_code", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "status_code", + "got": "status_code", + "got_raw": "status_code", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "_MIN_LENGTH", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "b''", + "got": "b''", + "got_raw": "b''", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "b''", + "got": "b'{\"query\":", + "got_raw": "b'{\"query\": \"whatever\"}'", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "b''", + "got": "b'{\"detail\":", + "got_raw": "b'{\"detail\": \"Redirecting...\"}'", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "b''", + "got": "b''", + "got_raw": "b''", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "b''", + "got": "b''", + "got_raw": "b''", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "b''", + "got": "b''", + "got_raw": "b''", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "b''", + "got": "b''", + "got_raw": "b''", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "'field'", + "got": "\"extra\"", + "got_raw": "\"extra\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "'HS256'", + "got": "'HS256'", + "got_raw": "'HS256'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "'authed'", + "got": "snapshot({", + "got_raw": "snapshot({ 'status': 'authed' })", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "'authed'", + "got": "snapshot({})", + "got_raw": "snapshot({})", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.2 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "'text/html'", + "got": "'application/json'", + "got_raw": "'application/json'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.20689655172413793 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "'not authed'", + "got": "{ 'status': 'ok'", + "got_raw": "{ 'status': 'ok', }", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "'CustomValue'", + "got": "'Custom-Value'", + "got_raw": "'Custom-Value'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9629629629629629 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "'value_error'", + "got": "'json'", + "got_raw": "'json'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3157894736842105 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "'value_error'", + "got": "'ValidationError'", + "got_raw": "'ValidationError'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "'none handled'", + "got": "snapshot({})", + "got_raw": "snapshot({})", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "'Error message'", + "got": "snapshot({ 'error': 'Invalid data' })", + "got_raw": "snapshot({ 'error': 'Invalid data' })", + "exact_match": false, + "code_bleu": 0.13485111859503685, + "edit_similarity": 0.23076923076923078 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "'customGetUser'", + "got": "'get_whatever'", + "got_raw": "'get_whatever'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.41379310344827586 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "'{\"key\":\"value\"}'", + "got": "request_data['extra']", + "got_raw": "request_data['extra']", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.21052631578947367 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot(1)", + "got": "{", + "got_raw": "{ 'error': 'Conflict', }", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "list(expected)", + "got": "expected", + "got_raw": "expected", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot(['a'])", + "got": "snapshot({", + "got_raw": "snapshot({ 'hello': 'world' })", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.72 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "len(operation_ids)", + "got": "len(operation_ids)", + "got_raw": "len(operation_ids)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "request_data.keys()", + "got": "request_data.keys()", + "got_raw": "request_data.keys()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "str(exc_info.value)", + "got": "str(exc_info)", + "got_raw": "str(exc_info)", + "exact_match": false, + "code_bleu": 0.4714045207910316, + "edit_similarity": 0.8125 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "sorted(request_data)", + "got": "sorted(request_data)", + "got_raw": "sorted(request_data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot({'jwt': []})", + "got": "[(\"bearer\"", + "got_raw": "[(\"bearer\", [])]", + "exact_match": false, + "code_bleu": 0.16666666666666663, + "edit_similarity": 0.06451612903225806 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot({'field': 1})", + "got": "snapshot({ 'field': '1' })", + "got_raw": "snapshot({ 'field': '1' })", + "exact_match": false, + "code_bleu": 0.6606328636027614, + "edit_similarity": 0.9166666666666666 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot(b'{\"a\": \"b\"}')", + "got": "b'{\"root\": {\"a\": \"b\"}}'", + "got_raw": "b'{\"root\": {\"a\": \"b\"}}'", + "exact_match": false, + "code_bleu": 0.2149139863647084, + "edit_similarity": 0.5652173913043478 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "len(request_data['user'])", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.07142857142857142, + "edit_similarity": 0.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot({'http_basic': []})", + "got": "[(\"basic\"", + "got_raw": "[(\"basic\", [])]", + "exact_match": false, + "code_bleu": 0.16666666666666663, + "edit_similarity": 0.32432432432432434 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "_QueryModel(filter='python')", + "got": "{'limit':", + "got_raw": "{'limit': '10'}", + "exact_match": false, + "code_bleu": 0.18995892141289814, + "edit_similarity": 0.16216216216216217 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "_HeaderModel(whatever='yes')", + "got": "_HeaderModel()", + "got_raw": "_HeaderModel()", + "exact_match": false, + "code_bleu": 0.3194715521231362, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "dataclasses.asdict(decoded_token)", + "got": "dataclasses.asdict(decoded_token)", + "got_raw": "dataclasses.asdict(decoded_token)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot({'format': 'invalid-format'})", + "got": "snapshot({})", + "got_raw": "snapshot({})", + "exact_match": false, + "code_bleu": 0.377671924669596, + "edit_similarity": 0.48 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "response.content.decode('utf-8').lower()", + "got": "response.content", + "got_raw": "response.content", + "exact_match": false, + "code_bleu": 0.21022410381342863, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "EndpointMetadataError, match=str(status))", + "got": "EndpointMetadataError)", + "got_raw": "EndpointMetadataError)", + "exact_match": false, + "code_bleu": 0.1321341238891912, + "edit_similarity": 0.6984126984126984 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot( {'message': 'Page not found'}, )", + "got": "snapshot({ 'error': 'Not Found', 'message': 'Exception' })", + "got_raw": "snapshot({ 'error': 'Not Found', 'message': 'Exception' })", + "exact_match": false, + "code_bleu": 0.2545093860020286, + "edit_similarity": 0.58 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot({ 'email': email, 'age': user_id, })", + "got": "snapshot({ 'email': email, 'age': faker.unique.random_int() })", + "got_raw": "snapshot({ 'email': email, 'age': faker.unique.random_int() })", + "exact_match": false, + "code_bleu": 0.5184341074271375, + "edit_similarity": 0.7663551401869159 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot({ 'django_session': [], 'csrf': [], })", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.09428090415820634, + "edit_similarity": 0.08163265306122448 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot({ 'detail': [{'msg': 'API Error'}], })", + "got": "{ 'error': 'API Error', }", + "got_raw": "{ 'error': 'API Error', }", + "exact_match": false, + "code_bleu": 0.15304117665113792, + "edit_similarity": 0.5833333333333334 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot( {'lax_field': 1, 'strict_field': 1}, )", + "got": "snapshot({ 'lax_field': '1', 'strict_field': 1, })", + "got_raw": "snapshot({ 'lax_field': '1', 'strict_field': 1, })", + "exact_match": false, + "code_bleu": 0.6076795808137692, + "edit_similarity": 0.9387755102040817 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot({ 'error': [{'message': 'test msg'}], })", + "got": "{ 'error': 'Payment required', 'error_type': ErrorType.PAYMENT_REQUIRED, }", + "got_raw": "{ 'error': 'Payment required', 'error_type': ErrorType.PAYMENT_REQUIRED, }", + "exact_match": false, + "code_bleu": 0.19120871059622507, + "edit_similarity": 0.35772357723577236 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot( {'first_field': 1, 'second_field': 1}, )", + "got": "snapshot({ 'first_field': '1', 'second_field': 1, })", + "got_raw": "snapshot({ 'first_field': '1', 'second_field': 1, })", + "exact_match": false, + "code_bleu": 0.6076795808137692, + "edit_similarity": 0.9411764705882353 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot({ 'detail': [{'msg': 'Internal server error'}], })", + "got": "snapshot({ 'error': 'Invalid type: . Expected: .' })", + "got_raw": "snapshot({ 'error': 'Invalid type: . Expected: .' })", + "exact_match": false, + "code_bleu": 0.1967434523911683, + "edit_similarity": 0.2978723404255319 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot({ 'detail': [{'msg': 'Internal server error'}], })", + "got": "snapshot({ 'error': 'Unable to serialize response body.' })", + "got_raw": "snapshot({ 'error': 'Unable to serialize response body.' })", + "exact_match": false, + "code_bleu": 0.1967434523911683, + "edit_similarity": 0.3559322033898305 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot( 'Set-Cookie: error_id=value; Path=/; SameSite=lax', )", + "got": "''", + "got_raw": "''", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.06153846153846154 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot( 'Set-Cookie: optional=123; Path=/; SameSite=strict', )", + "got": "''", + "got_raw": "''", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.06060606060606061 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot({ 'detail': [{'msg': IsStr(), 'type': 'not_allowed'}], })", + "got": "{ 'error': 'Method not allowed' }", + "got_raw": "{ 'error': 'Method not allowed' }", + "exact_match": false, + "code_bleu": 0.07730188463692832, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "snapshot( {'detail': [{'msg': 'Page not found', 'type': 'not_found'}]}, )", + "got": "snapshot({ 'error': 'Not Found', 'error_type': ErrorType.NOT_FOUND, })", + "got_raw": "snapshot({ 'error': 'Not Found', 'error_type': ErrorType.NOT_FOUND, })", + "exact_match": false, + "code_bleu": 0.15198642974521234, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "1", + "got": "snapshot({", + "got_raw": "snapshot({ 'error': '1', 'status_code': HTTPStatus.PAYMENT_REQUIRED.value, })", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "1", + "got": "{'test':", + "got_raw": "{'test': '123'}", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "1", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "1", + "got": "{'schema_only':", + "got_raw": "{'schema_only': True}", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/django-modern-rest", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + } + ] +} \ No newline at end of file