diff --git "a/hypernet_paw/no_oracle_results/ir_test/results.json" "b/hypernet_paw/no_oracle_results/ir_test/results.json" new file mode 100644--- /dev/null +++ "b/hypernet_paw/no_oracle_results/ir_test/results.json" @@ -0,0 +1,47008 @@ +{ + "exact_match_pct": 65.83684412102643, + "exact_match_count": 3438, + "n": 5222, + "n_total": 5222, + "code_bleu": 0.7902215301971106, + "edit_similarity": 0.8043026757317001, + "entries": [ + { + "repo": "0xricksanchez/like-dbg", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "0xricksanchez/like-dbg", + "expected": "\"b1f3f40af1e5c64dbf4a1302ca94ac12\"", + "got": "\"00000000000000000000000000000000\"", + "got_raw": "\"00000000000000000000000000000000\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "0xricksanchez/like-dbg", + "expected": "\"5.15.67\"", + "got": "\"mmp\"", + "got_raw": "\"mmp\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "0xricksanchez/like-dbg", + "expected": "\"https://cdn.kernel.org/pub/linux/kernel/vKMAJOR.x/linux-KMAJOR.KMINOR.KPATCH.tar.xz\"", + "got": "\"https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/mmp/\"", + "got_raw": "\"https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/mmp/\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6075949367088608 + }, + { + "repo": "0xricksanchez/like-dbg", + "expected": "-2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "0xricksanchez/like-dbg", + "expected": "lkdl.archive", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.25 + }, + { + "repo": "0xricksanchez/like-dbg", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "0xricksanchez/like-dbg", + "expected": "f\"https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/snapshot/linux-{lkdl.commit}.tar.gz\"", + "got": "dl_uri", + "got_raw": "dl_uri", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.05405405405405406 + }, + { + "repo": "0xricksanchez/like-dbg", + "expected": "SystemExit", + "got": "SystemExit", + "got_raw": "SystemExit", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "AlignmentResearch/tuned-lens", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "AlignmentResearch/tuned-lens", + "expected": "tokens[:, :1]", + "got": "tokens[:, 0]", + "got_raw": "tokens[:, 0]", + "exact_match": false, + "code_bleu": 0.55153618642351, + "edit_similarity": 0.88 + }, + { + "repo": "AlignmentResearch/tuned-lens", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "AlignmentResearch/tuned-lens", + "expected": "our_js_bwd)", + "got": "our_js_bwd)", + "got_raw": "our_js_bwd)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "AndrewAnnex/SpiceyPy", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "AndrewAnnex/SpiceyPy", + "expected": "BaseException)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4 + }, + { + "repo": "AndrewAnnex/SpiceyPy", + "expected": "np.pi", + "got": "np.pi", + "got_raw": "np.pi", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "AndrewAnnex/SpiceyPy", + "expected": "\"some error occured\"", + "got": "\"some error occured\"", + "got_raw": "\"some error occured\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "AndrewAnnex/SpiceyPy", + "expected": "19", + "got": "30", + "got_raw": "30", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "AnonymouX47/term-image", + "expected": "width", + "got": "width", + "got_raw": "width", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "AnonymouX47/term-image", + "expected": "TypeError, match=r\"'filepath'\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.153509990260506, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "AnonymouX47/term-image", + "expected": "9", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "AnonymouX47/term-image", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "AnonymouX47/term-image", + "expected": "b\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.8 + }, + { + "repo": "AnonymouX47/term-image", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "AnonymouX47/term-image", + "expected": "1", + "got": "RenderArgs(Bar)", + "got_raw": "RenderArgs(Bar)", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.0 + }, + { + "repo": "AnonymouX47/term-image", + "expected": "f\"{self.trans:1.1", + "got": "str(self.trans)", + "got_raw": "str(self.trans)", + "exact_match": false, + "code_bleu": 0.30835021267538976, + "edit_similarity": 0.625 + }, + { + "repo": "AnonymouX47/term-image", + "expected": "_size", + "got": "_size", + "got_raw": "_size", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "AnonymouX47/term-image", + "expected": "{}", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "AnonymouX47/term-image", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "AnonymouX47/term-image", + "expected": "100", + "got": "100", + "got_raw": "100", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "AnonymouX47/term-image", + "expected": "\"", + "got": "\"", + "got_raw": "\"#\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "AnonymouX47/term-image", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "AnonymouX47/term-image", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Azure-Samples/rag-postgres-openai-python", + "expected": "\"22\"", + "got": "\"0\"", + "got_raw": "\"0\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "Azure-Samples/rag-postgres-openai-python", + "expected": "\"\"", + "got": "\"token\"", + "got_raw": "\"token\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "Azure-Samples/rag-postgres-openai-python", + "expected": "\"text-embedding-3-large\"", + "got": "\"text-embedding-ada-002\"", + "got_raw": "\"text-embedding-ada-002\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "Azure-Samples/rag-postgres-openai-python", + "expected": "\"45\"", + "got": "str(len(json.dumps({\"error\": \"Item not found\"})))", + "got_raw": "str(len(json.dumps({\"error\": \"Item not found\"})))", + "exact_match": false, + "code_bleu": 0.06468490584192432, + "edit_similarity": 0.07547169811320754 + }, + { + "repo": "Azure-Samples/rag-postgres-openai-python", + "expected": "response.content", + "got": "response.data", + "got_raw": "response.data", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6896551724137931 + }, + { + "repo": "Azure-Samples/rag-postgres-openai-python", + "expected": "\"vectors\"", + "got": "\"default\"", + "got_raw": "\"default\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "BoboTiG/python-mss", + "expected": "SystemExit)", + "got": "SystemExit)", + "got_raw": "SystemExit)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BoboTiG/python-mss", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BoboTiG/python-mss", + "expected": "repr(ref1)", + "got": "repr(ref1)", + "got_raw": "repr(ref1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BoboTiG/python-mss", + "expected": "output", + "got": "output", + "got_raw": "output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BoboTiG/python-mss", + "expected": "im2.rgb", + "got": "im2.rgb", + "got_raw": "im2.rgb", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BoboTiG/python-mss", + "expected": "ScreenShotError)", + "got": "ScreenShotError)", + "got_raw": "ScreenShotError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BoboTiG/python-mss", + "expected": "ScreenShotError, match=r\"\\b8\\b\")", + "got": "ScreenShotError, match=\"8-bit color depth is not supported\")", + "got_raw": "ScreenShotError, match=\"8-bit color depth is not supported\")", + "exact_match": false, + "code_bleu": 0.55153618642351, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "BoboTiG/python-mss", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BoboTiG/python-mss", + "expected": "1024", + "got": "1024", + "got_raw": "1024", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BrainBlend-AI/atomic-agents", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BrainBlend-AI/atomic-agents", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BrainBlend-AI/atomic-agents", + "expected": "Any", + "got": "\"Any\"", + "got_raw": "\"Any\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "BrainBlend-AI/atomic-agents", + "expected": "expected_prompt", + "got": "expected_prompt", + "got_raw": "expected_prompt", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BrainBlend-AI/atomic-agents", + "expected": "30)", + "got": "30)", + "got_raw": "30)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BrainBlend-AI/atomic-agents", + "expected": "payload_sent[\"include\"]", + "got": "payload_sent[\"include\"]", + "got_raw": "payload_sent[\"include\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BrainBlend-AI/atomic-agents", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BrainBlend-AI/atomic-agents", + "expected": "result2", + "got": "result2", + "got_raw": "result2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BrainBlend-AI/atomic-agents", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BrainBlend-AI/atomic-agents", + "expected": "steps", + "got": "steps", + "got_raw": "steps", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BrainBlend-AI/atomic-agents", + "expected": "\"Test description\"", + "got": "\"Test description\"", + "got_raw": "\"Test description\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BrainBlend-AI/atomic-agents", + "expected": "\"MockToolCall\"", + "got": "\"MockToolCall\"", + "got_raw": "\"MockToolCall\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "BrainBlend-AI/atomic-agents", + "expected": "CustomInput", + "got": "CustomInput", + "got_raw": "CustomInput", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "8002", + "got": "8002", + "got_raw": "8002", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "initial_models", + "got": "initial_models", + "got_raw": "initial_models", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "8001", + "got": "8001", + "got_raw": "8001", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "{}", + "got": "result[\"metadata\"][\"quantization\"]", + "got_raw": "result[\"metadata\"][\"quantization\"]", + "exact_match": false, + "code_bleu": 0.1561969968460128, + "edit_similarity": 0.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "8080", + "got": "8080", + "got_raw": "8080", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"gguf\"", + "got": "\"gguf\"", + "got_raw": "\"gguf\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Chen-zexi/vllm-cli", + "expected": "\"stop\"", + "got": "\"stop\"", + "got_raw": "\"stop\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cloxl/xhshow", + "expected": "len(uri)", + "got": "len(uri)", + "got_raw": "len(uri)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cloxl/xhshow", + "expected": "session.window_props_length", + "got": "session.window_props_length", + "got_raw": "session.window_props_length", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cloxl/xhshow", + "expected": "50", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "Cloxl/xhshow", + "expected": "\"/api/search\"", + "got": "'/api/search'", + "got_raw": "'/api/search'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8461538461538461 + }, + { + "repo": "Cloxl/xhshow", + "expected": "32", + "got": "32", + "got_raw": "32", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "0", + "got": "[\"latest(nonexistent)>=1\"]", + "got_raw": "[\"latest(nonexistent)>=1\"]", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "result.output", + "got": "result.output", + "got_raw": "result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "n", + "got": "n", + "got_raw": "n", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "out", + "got": "out", + "got_raw": "out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "ep", + "got": "ep", + "got_raw": "ep", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "d", + "got": "d", + "got_raw": "d", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "\"b\"", + "got": "\"b\"", + "got_raw": "\"b\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "h", + "got": "h", + "got_raw": "h", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "1", + "got": "[\"health_score>=70\"]", + "got_raw": "[\"health_score>=70\"]", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "f", + "got": "f", + "got_raw": "f", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Cranot/roam-code", + "expected": "7", + "got": "7", + "got_raw": "7", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "CursorTouch/Windows-MCP", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "CursorTouch/Windows-MCP", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "CursorTouch/Windows-MCP", + "expected": "1900", + "got": "1920", + "got_raw": "1920", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "CursorTouch/Windows-MCP", + "expected": "\"Maximized\"", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "CursorTouch/Windows-MCP", + "expected": "\"''\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "CursorTouch/Windows-MCP", + "expected": "MAX_RETRIES", + "got": "MAX_RETRIES", + "got_raw": "MAX_RETRIES", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "CursorTouch/Windows-MCP", + "expected": "1920", + "got": "2000", + "got_raw": "2000", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "CursorTouch/Windows-MCP", + "expected": "{\"key\": \"value\", \"count\": 42}", + "got": "{\"key\": \"value\", \"count\": 42}", + "got_raw": "{\"key\": \"value\", \"count\": 42}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DHI/terracotta", + "expected": "np.array([2, 0, 1]))", + "got": "[1, 0, 1])", + "got_raw": "[1, 0, 1])", + "exact_match": false, + "code_bleu": 0.46192237153867044, + "edit_similarity": 0.6 + }, + { + "repo": "DHI/terracotta", + "expected": "db", + "got": "db", + "got_raw": "db", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DHI/terracotta", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DHI/terracotta", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DHI/terracotta", + "expected": "tile_identity_transform)", + "got": "tile_identity_transform)", + "got_raw": "tile_identity_transform)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DHI/terracotta", + "expected": "src2.read())", + "got": "src2.read())", + "got_raw": "src2.read())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DHI/terracotta", + "expected": "(\"foo\", \"rgb\")", + "got": "[\"rgb\", \"foo\"]", + "got_raw": "[\"rgb\", \"foo\"]", + "exact_match": false, + "code_bleu": 0.32466791547509893, + "edit_similarity": 0.35714285714285715 + }, + { + "repo": "DHI/terracotta", + "expected": "repr(db)", + "got": "repr(db)", + "got_raw": "repr(db)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DHI/terracotta", + "expected": "driver.key_names", + "got": "tuple(", + "got_raw": "tuple(\n driver.keys()", + "exact_match": false, + "code_bleu": 0.30120066786994826, + "edit_similarity": 0.09090909090909091 + }, + { + "repo": "DHI/terracotta", + "expected": "keys", + "got": "keys", + "got_raw": "keys", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DHI/terracotta", + "expected": "test_tile_size", + "got": "test_tile_size", + "got_raw": "test_tile_size", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DHI/terracotta", + "expected": "terracotta.cmaps.get_cmaps.AVAILABLE_CMAPS", + "got": "terracotta.cmaps.get_cmaps.get_cmaps()", + "got_raw": "terracotta.cmaps.get_cmaps.get_cmaps()", + "exact_match": false, + "code_bleu": 0.6389431042462724, + "edit_similarity": 0.7 + }, + { + "repo": "DHI/terracotta", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DHI/terracotta", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DHI/terracotta", + "expected": "terracotta.exceptions.TileOutOfBoundsError)", + "got": "exceptions.TerracottaError)", + "got_raw": "exceptions.TerracottaError)", + "exact_match": false, + "code_bleu": 0.3387551654364098, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "DHI/terracotta", + "expected": "result)", + "got": "result)", + "got_raw": "result)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DLR-RM/stable-baselines3", + "expected": "1.0", + "got": "1.0", + "got_raw": "1.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DLR-RM/stable-baselines3", + "expected": "1.0", + "got": "1.0", + "got_raw": "1.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DLR-RM/stable-baselines3", + "expected": "0.8", + "got": "0.8", + "got_raw": "0.8", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DLR-RM/stable-baselines3", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DLR-RM/stable-baselines3", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DLR-RM/stable-baselines3", + "expected": "1.0", + "got": "0.5", + "got_raw": "0.5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "DLR-RM/stable-baselines3", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DLR-RM/stable-baselines3", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DLR-RM/stable-baselines3", + "expected": "str(tmp_path)", + "got": "str(tmp_path)", + "got_raw": "str(tmp_path)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DLR-RM/stable-baselines3", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DLR-RM/stable-baselines3", + "expected": "infos[0][\"episode\"][\"l\"]", + "got": "infos[0][\"episode\"][\"l\"]", + "got_raw": "infos[0][\"episode\"][\"l\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DLR-RM/stable-baselines3", + "expected": "gamma", + "got": "gamma", + "got_raw": "gamma", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DLR-RM/stable-baselines3", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DLR-RM/stable-baselines3", + "expected": "0.3", + "got": "0.3", + "got_raw": "0.3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DLR-RM/stable-baselines3", + "expected": "model.policy.device.type", + "got": "model.device.type", + "got_raw": "model.device.type", + "exact_match": false, + "code_bleu": 0.6006402966097961, + "edit_similarity": 0.8292682926829268 + }, + { + "repo": "DLR-RM/stable-baselines3", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DLR-RM/stable-baselines3", + "expected": "AssertionError, match=r\"use_sde=True\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.153509990260506, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "DLR-RM/stable-baselines3", + "expected": "callback.locals[\"new_obs\"]", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.08333333333333333, + "edit_similarity": 0.2 + }, + { + "repo": "DebarghaG/proofofthought", + "expected": "any(\"Unknown optimization type\" in msg for msg in cm.output))", + "got": "any(\"Unknown objective type\" in msg for msg in cm.output))", + "got_raw": "any(\"Unknown objective type\" in msg for msg in cm.output))", + "exact_match": false, + "code_bleu": 0.8091067115702212, + "edit_similarity": 0.8739495798319328 + }, + { + "repo": "DebarghaG/proofofthought", + "expected": "any(\"dictionary\" in msg for msg in cm.output))", + "got": "any(\"Invalid constants structure\" in msg for msg in cm.output))", + "got_raw": "any(\"Invalid constants structure\" in msg for msg in cm.output))", + "exact_match": false, + "code_bleu": 0.8091067115702212, + "edit_similarity": 0.7706422018348624 + }, + { + "repo": "DebarghaG/proofofthought", + "expected": "str(ctx.exception))", + "got": "str(ctx.exception))", + "got_raw": "str(ctx.exception))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DebarghaG/proofofthought", + "expected": "20000)", + "got": "20000)", + "got_raw": "20000)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DebarghaG/proofofthought", + "expected": "5)", + "got": "5)", + "got_raw": "5)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DebarghaG/proofofthought", + "expected": "str(ctx.exception))", + "got": "str(ctx.exception))", + "got_raw": "str(ctx.exception))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Delgan/loguru", + "expected": "\"\"", + "got": "expected", + "got_raw": "expected", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "Delgan/loguru", + "expected": "file.read()", + "got": "file.read()", + "got_raw": "file.read()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Delgan/loguru", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Delgan/loguru", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Delgan/loguru", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Delgan/loguru", + "expected": "OSError, match=\"^Rotation error$\")", + "got": "OSError)", + "got_raw": "OSError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.38095238095238093 + }, + { + "repo": "Delgan/loguru", + "expected": "\"2 3\\n\"", + "got": "\"2 3\\n\"", + "got_raw": "\"2 3\\n\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Delgan/loguru", + "expected": "\"log.log\"", + "got": "format", + "got_raw": "format", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "Delgan/loguru", + "expected": "ValueError, match=r\".* most likely a mistake\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.16988673071467442, + "edit_similarity": 0.38596491228070173 + }, + { + "repo": "Delgan/loguru", + "expected": "\"A\\nB\\n\"", + "got": "\"A\\nB\\n\"", + "got_raw": "\"A\\nB\\n\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Delgan/loguru", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Delgan/loguru", + "expected": "\"1\\n\"", + "got": "\"1\\n\"", + "got_raw": "\"1\\n\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Delgan/loguru", + "expected": "\"12 Test\\n\"", + "got": "\"10 Test\\n\"", + "got_raw": "\"10 Test\\n\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "Delgan/loguru", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Delgan/loguru", + "expected": "\"0 A\\n\"", + "got": "\"0 A\\n\"", + "got_raw": "\"0 A\\n\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Delgan/loguru", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Delgan/loguru", + "expected": "\"\u5929\\n\"", + "got": "\"\u5929\\n\"", + "got_raw": "\"\u5929\\n\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Delgan/loguru", + "expected": "dict(maj=\"ABC!DEF\")", + "got": "{\"maj\": \"ABC!DEF\"}", + "got_raw": "{\"maj\": \"ABC!DEF\"}", + "exact_match": false, + "code_bleu": 0.22751007189242223, + "edit_similarity": 0.6486486486486487 + }, + { + "repo": "Delgan/loguru", + "expected": "\"1 2\\n\"", + "got": "\"1 2\\n\"", + "got_raw": "\"1 2\\n\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Delgan/loguru", + "expected": "error", + "got": "error", + "got_raw": "error", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DisnakeDev/disnake", + "expected": "TypeError, match=r\"id parameter must be convertible to int not \")", + "got": "pytest.raises(ValueError)", + "got_raw": "pytest.raises(ValueError)", + "exact_match": false, + "code_bleu": 0.18933935257849616, + "edit_similarity": 0.1553398058252427 + }, + { + "repo": "DisnakeDev/disnake", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DisnakeDev/disnake", + "expected": "1024", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "DisnakeDev/disnake", + "expected": "\"a group\"", + "got": "\"group\"", + "got_raw": "\"group\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.875 + }, + { + "repo": "DisnakeDev/disnake", + "expected": "0", + "got": "obj.custom_id", + "got_raw": "obj.custom_id", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "DisnakeDev/disnake", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DisnakeDev/disnake", + "expected": "frozenset([456, 789])", + "got": "[\"456\", \"789\"]", + "got_raw": "[\"456\", \"789\"]", + "exact_match": false, + "code_bleu": 0.20291744717193683, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "DisnakeDev/disnake", + "expected": "CustomCommand)", + "got": "CustomCommand)", + "got_raw": "CustomCommand)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DisnakeDev/disnake", + "expected": "high", + "got": "high", + "got_raw": "high", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DisnakeDev/disnake", + "expected": "cog.group", + "got": "cog.group", + "got_raw": "cog.group", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DisnakeDev/disnake", + "expected": "210174600000", + "got": "snowflake", + "got_raw": "snowflake", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "DisnakeDev/disnake", + "expected": "Permissions(32)", + "got": "32", + "got_raw": "32", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "DisnakeDev/disnake", + "expected": "5", + "got": "7", + "got_raw": "7", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "DisnakeDev/disnake", + "expected": "exception)", + "got": "exception)", + "got_raw": "exception)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "DisnakeDev/disnake", + "expected": "value", + "got": "value", + "got_raw": "value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "EbodShojaei/bake", + "expected": "expected_lines", + "got": "expected_lines", + "got_raw": "expected_lines", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "EbodShojaei/bake", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "EbodShojaei/bake", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "EbodShojaei/bake", + "expected": "[\"VAR:=value\", \"VAR2=value\"]", + "got": "[\"VAR=value\", \"VAR2=value\"]", + "got_raw": "[\"VAR=value\", \"VAR2=value\"]", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.9818181818181818 + }, + { + "repo": "EbodShojaei/bake", + "expected": "fish_script", + "got": "fish_script", + "got_raw": "fish_script", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "EbodShojaei/bake", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "EbodShojaei/bake", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "EbodShojaei/bake", + "expected": "\"LONGER_NAME = value\"", + "got": "\"LONGER_NAME = value\"", + "got_raw": "\"LONGER_NAME = value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "EbodShojaei/bake", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "EbodShojaei/bake", + "expected": "formatted_lines", + "got": "formatted_lines", + "got_raw": "formatted_lines", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "FinanceData/FinanceDataReader", + "expected": "200", + "got": "1000", + "got_raw": "1000", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "FinanceData/FinanceDataReader", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "FinanceData/FinanceDataReader", + "expected": "90", + "got": "900", + "got_raw": "900", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8 + }, + { + "repo": "Forethought-Technologies/AutoChain", + "expected": "\"User: user query\\nAssistant: response to user\\n\"", + "got": "\"User: user query\\nAI: response to user\\n\"", + "got_raw": "\"User: user query\\nAI: response to user\\n\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9010989010989011 + }, + { + "repo": "Forethought-Technologies/AutoChain", + "expected": "{ \"name\": \"sample_tool_func_with_type\", \"description\": \"This is just a dummy tool with typing info\", \"parameters\": { \"type\": \"object\", \"properties\": {\"k\": {\"type\": \"int\"}}, \"required\": [\"k\"], }, }", + "got": "{", + "got_raw": "{\n \"name\": \"sample_tool_func_with_type\",\n \"description\": \"This is just a \" \"dummy tool with typing info\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\"k\": {\"type\": \"integer\"}},\n \"required\": [\"k\"],\n },\n }", + "exact_match": false, + "code_bleu": 0.016070366418955692, + "edit_similarity": 0.01015228426395939 + }, + { + "repo": "Forethought-Technologies/AutoChain", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Forethought-Technologies/AutoChain", + "expected": "text", + "got": "text", + "got_raw": "text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Forethought-Technologies/AutoChain", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "caplog.text", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "target", + "got": "target", + "got_raw": "target", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "instance", + "got": "instance", + "got_raw": "instance", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "result.output", + "got": "result.output", + "got_raw": "result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "set()", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.30120066786994826, + "edit_similarity": 0.0 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "result.output", + "got": "result.output", + "got_raw": "result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "\"1.1\"", + "got": "\"1.1\"", + "got_raw": "\"1.1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "captured.err", + "got": "captured.err", + "got_raw": "captured.err", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "result.stdout", + "got": "result.output", + "got_raw": "result.output", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "message", + "got": "message", + "got_raw": "message", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "result.stdout", + "got": "result.stdout", + "got_raw": "result.stdout", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "result)", + "got": "result )", + "got_raw": " result )", + "exact_match": false, + "code_bleu": 0.7071067811865476, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "result.output", + "got": "result.output", + "got_raw": "result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "GitGuardian/ggshield", + "expected": "valid", + "got": "valid", + "got_raw": "valid", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "[1, 2, 3]", + "got": "[1, 2, 3]", + "got_raw": "[1, 2, 3]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "obs2", + "got": "obs2", + "got_raw": "obs2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "{}", + "got": "second[0]", + "got_raw": "second[0]", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "(2, 25)", + "got": "(2, 2)", + "got_raw": "(2, 2)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "IDSIA/sacred", + "expected": "[1, 2]", + "got": "[1, 2]", + "got_raw": "[1, 2]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"t\"", + "got": "\"t\"", + "got_raw": "\"t\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "tmpfile.content", + "got": "tmpfile.read()", + "got_raw": "tmpfile.read()", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.6206896551724138 + }, + { + "repo": "IDSIA/sacred", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "\"b.d\"", + "got": "\"b.d\"", + "got_raw": "\"b.d\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "1", + "got": "0.1", + "got_raw": "0.1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "IDSIA/sacred", + "expected": "{\"d\"}", + "got": "{\"d\"}", + "got_raw": "{\"d\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "cf.run", + "got": "cf.run", + "got_raw": "cf.run", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "{\"i\": 10}", + "got": "{\"i\": 10}", + "got_raw": "{\"i\": 10}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "random.Random", + "got": "opt.np.random.RandomState", + "got_raw": "opt.np.random.RandomState", + "exact_match": false, + "code_bleu": 0.24446151121745047, + "edit_similarity": 0.6842105263157895 + }, + { + "repo": "IDSIA/sacred", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "(nr1, seed1)", + "got": "(nr1, seed1)", + "got_raw": "(nr1, seed1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "seed2", + "got": "seed2", + "got_raw": "seed2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "5", + "got": "4", + "got_raw": "4", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "IDSIA/sacred", + "expected": "foo", + "got": "foo", + "got_raw": "foo", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JWock82/Pynite", + "expected": "initial_nodes", + "got": "initial_nodes", + "got_raw": "initial_nodes", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JWock82/Pynite", + "expected": "{})", + "got": "{})", + "got_raw": "{})", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JWock82/Pynite", + "expected": "model.springs", + "got": "model.springs", + "got_raw": "model.springs", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JWock82/Pynite", + "expected": "110)", + "got": "110)", + "got_raw": "110)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JWock82/Pynite", + "expected": "2)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "JWock82/Pynite", + "expected": "0.03", + "got": "0.03", + "got_raw": "0.03", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JWock82/Pynite", + "expected": "{})", + "got": "{})", + "got_raw": "{})", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JWock82/Pynite", + "expected": "29000)", + "got": "29000)", + "got_raw": "29000)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JWock82/Pynite", + "expected": "model.nodes", + "got": "model.nodes", + "got_raw": "model.nodes", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JWock82/Pynite", + "expected": "1.2)", + "got": "1.2)", + "got_raw": "1.2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JWock82/Pynite", + "expected": "18", + "got": "18", + "got_raw": "18", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JWock82/Pynite", + "expected": "-2.5", + "got": "-1.0", + "got_raw": "-1.0", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "JWock82/Pynite", + "expected": "0.05)", + "got": "0.03)", + "got_raw": "0.03)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8 + }, + { + "repo": "JoshuaC215/agent-service-toolkit", + "expected": "\"What is the weather in Tokyo?\"", + "got": "\"Tokyo?\"", + "got_raw": "\"Tokyo?\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.41025641025641024 + }, + { + "repo": "JoshuaC215/agent-service-toolkit", + "expected": "NotImplementedError, match=\"Deepgram STT provider not yet implemented\")", + "got": "NotImplementedError)", + "got_raw": "NotImplementedError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.43956043956043955 + }, + { + "repo": "JoshuaC215/agent-service-toolkit", + "expected": "RUN_ID", + "got": "RUN_ID", + "got_raw": "RUN_ID", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JoshuaC215/agent-service-toolkit", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JoshuaC215/agent-service-toolkit", + "expected": "fake_store", + "got": "fake_store", + "got_raw": "fake_store", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JoshuaC215/agent-service-toolkit", + "expected": "\"claude-haiku-4-5\"", + "got": "\"claude-2\"", + "got_raw": "\"claude-2\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6428571428571429 + }, + { + "repo": "JoshuaC215/agent-service-toolkit", + "expected": "logging.ERROR", + "got": "logging.ERROR", + "got_raw": "logging.ERROR", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JoshuaC215/agent-service-toolkit", + "expected": "\"ai\"", + "got": "\"ai\"", + "got_raw": "\"ai\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JoshuaC215/agent-service-toolkit", + "expected": "[\"This is a test response from the fake model.\"]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.39640237166757364, + "edit_similarity": 0.08 + }, + { + "repo": "JoshuaC215/agent-service-toolkit", + "expected": "\"42\"", + "got": "\"42\"", + "got_raw": "\"42\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JoshuaC215/agent-service-toolkit", + "expected": "LogLevel.DEBUG", + "got": "LogLevel.DEBUG", + "got_raw": "LogLevel.DEBUG", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JoshuaC215/agent-service-toolkit", + "expected": "mock_graph", + "got": "mock_graph", + "got_raw": "mock_graph", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JuanBindez/pytubefix", + "expected": "'hZpzr8TbF08'", + "got": "'hZpzr8TbF08'", + "got_raw": "'hZpzr8TbF08'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JuanBindez/pytubefix", + "expected": "3399554", + "got": "123456789", + "got_raw": "123456789", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.25 + }, + { + "repo": "JuanBindez/pytubefix", + "expected": "playlist_html", + "got": "playlist_html", + "got_raw": "playlist_html", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JuanBindez/pytubefix", + "expected": "18", + "got": "18", + "got_raw": "18", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JuanBindez/pytubefix", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JuanBindez/pytubefix", + "expected": "'UCs6nmQViDpUw0nuIx9c_WvA'", + "got": "'UC1000000000000000000000'", + "got_raw": "'UC1000000000000000000000'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.19230769230769232 + }, + { + "repo": "JuanBindez/pytubefix", + "expected": "RegexMatchError)", + "got": "RegexMatchError)", + "got_raw": "RegexMatchError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JuanBindez/pytubefix", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JuanBindez/pytubefix", + "expected": "ytmd.raw_metadata[0]", + "got": "ytmd.raw_metadata", + "got_raw": "ytmd.raw_metadata", + "exact_match": false, + "code_bleu": 0.42044820762685725, + "edit_similarity": 0.918918918918919 + }, + { + "repo": "JuanBindez/pytubefix", + "expected": "{\"content-length\": \"16384\"}", + "got": "16384", + "got_raw": "16384", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.3125 + }, + { + "repo": "JuanBindez/pytubefix", + "expected": "\"abc\"", + "got": "\"abc\"", + "got_raw": "\"abc\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JuanBindez/pytubefix", + "expected": "{ 'foo': [], 'bar': {} }", + "got": "{'foo': [], 'bar': {}}", + "got_raw": "{'foo': [], 'bar': {}}", + "exact_match": false, + "code_bleu": 1.0, + "edit_similarity": 0.9565217391304348 + }, + { + "repo": "JuanBindez/pytubefix", + "expected": "'m8uHb5jIGN8'", + "got": "'m8uHb5jIGN8'", + "got_raw": "'m8uHb5jIGN8'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JuanBindez/pytubefix", + "expected": "0.02", + "got": "0.02", + "got_raw": "0.02", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "JuanBindez/pytubefix", + "expected": "'{\\'en\\': }'", + "got": "''", + "got_raw": "''", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7311827956989247 + }, + { + "repo": "Kludex/mangum", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Kludex/mangum", + "expected": "b\"\"", + "got": "b\"\"", + "got_raw": "b\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Kludex/mangum", + "expected": "{ \"statusCode\": 200, \"isBase64Encoded\": False, \"headers\": {\"content-type\": \"text/plain; charset=utf-8\"}, \"multiValueHeaders\": {}, \"body\": \"Hello, world!\", }", + "got": "b\"Hello, world!\"", + "got_raw": "b\"Hello, world!\"", + "exact_match": false, + "code_bleu": 0.039798886363834884, + "edit_similarity": 0.18604651162790697 + }, + { + "repo": "Kludex/mangum", + "expected": "LifespanFailure)", + "got": "LifespanFailure)", + "got_raw": "LifespanFailure)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Kludex/mangum", + "expected": "caplog.text", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Kludex/mangum", + "expected": "scope_body", + "got": "scope_body", + "got_raw": "scope_body", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Kludex/mangum", + "expected": "message", + "got": "message", + "got_raw": "message", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"protocol\": \"redis\", \"url\": \"redis://localhost:6379\", }", + "got": "{\"url\": \"http://localhost:8080\"}", + "got_raw": "{\"url\": \"http://localhost:8080\"}", + "exact_match": false, + "code_bleu": 0.2041241452319315, + "edit_similarity": 0.5393258426966292 + }, + { + "repo": "Lancetnik/Propan", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "\"hi\"", + "got": "message", + "got_raw": "message", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "broker", + "got": "broker", + "got_raw": "broker", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "RequestValidationError)", + "got": "RequestValidationError)", + "got_raw": "RequestValidationError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "4", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "\"test\")", + "got": "AnyCallable)", + "got_raw": "AnyCallable)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.21052631578947367 + }, + { + "repo": "Lancetnik/Propan", + "expected": "await test_broker.publish(\"\", queue, callback=True)", + "got": "await test_broker.send(queue, \"1\")", + "got_raw": "await test_broker.send(queue, \"1\")", + "exact_match": false, + "code_bleu": 0.2324002101948858, + "edit_similarity": 0.6588235294117647 + }, + { + "repo": "Lancetnik/Propan", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "{ \"Handler\": { \"bindings\": { \"redis\": { \"bindingVersion\": \"custom\", \"channel\": \"test\", \"method\": \"subscribe\", } }, \"servers\": [\"dev\"], \"subscribe\": { \"bindings\": {\"redis\": {\"bindingVersion\": \"custom\"}}, \"message\": {\"$ref\": \"", + "got": "[\"test\"]", + "got_raw": "[\"test\"]", + "exact_match": false, + "code_bleu": 0.030997209571832943, + "edit_similarity": 0.034482758620689655 + }, + { + "repo": "Lancetnik/Propan", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "\"FuncMessage\"", + "got": "\"func\"", + "got_raw": "\"func\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5263157894736842 + }, + { + "repo": "Lancetnik/Propan", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "Lancetnik/Propan", + "expected": "\"pong\"", + "got": "\"pong\"", + "got_raw": "\"pong\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "LonamiWebs/Telethon", + "expected": "[user]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.39640237166757364, + "edit_similarity": 0.5 + }, + { + "repo": "LonamiWebs/Telethon", + "expected": "unpickled_error.message", + "got": "unpickled_error.message", + "got_raw": "unpickled_error.message", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "LonamiWebs/Telethon", + "expected": "media_file)", + "got": "ynchronous_call.assert_called_once()", + "got_raw": "ynchronous_call.assert_called_once()", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.1702127659574468 + }, + { + "repo": "LonamiWebs/Telethon", + "expected": "sorted((e.offset, e.length) for e in entities_expected)", + "got": "(entities_expected)", + "got_raw": "(entities_expected)", + "exact_match": false, + "code_bleu": 0.11980183204617609, + "edit_similarity": 0.5135135135135135 + }, + { + "repo": "LonamiWebs/Telethon", + "expected": "[utils.get_input_peer(user)]", + "got": "[user]", + "got_raw": "[user]", + "exact_match": false, + "code_bleu": 0.2014818622169141, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "LonamiWebs/Telethon", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "\"images\")", + "got": "3)", + "got_raw": "3)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "\"texts\")", + "got": "\"text\")", + "got_raw": "\"text\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "\"popped\")", + "got": "\"hello\")", + "got_raw": "\"hello\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "\"final\")", + "got": "\"updated year\")", + "got_raw": "\"updated year\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.34782608695652173 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "\"1 day ago\"", + "got": "\"1 day ago\"", + "got_raw": "\"1 day ago\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "msg)", + "got": "msg)", + "got_raw": "msg)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "\"there\")", + "got": "\"there\")", + "got_raw": "\"there\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "\"now set\")", + "got": "\"saved\")", + "got_raw": "\"saved\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "[(2,)]", + "got": "[(2,)]", + "got_raw": "[(2,)]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "\"unknown\"", + "got": "\"unknown\"", + "got_raw": "\"unknown\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "\"no langs\")", + "got": "[\"English\"])", + "got_raw": "[\"English\"])", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.5217391304347826 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "\"11 GB\"", + "got": "\"11.23 GB\"", + "got_raw": "\"11.23 GB\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "\"deleted\")", + "got": "\"no saved\")", + "got_raw": "\"no saved\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "\"42\"", + "got": "42", + "got_raw": "42", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "\"(default)\"", + "got": "\"Default\"", + "got_raw": "\"Default\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8 + }, + { + "repo": "LuteOrg/lute-v3", + "expected": "t.text", + "got": "text.lower()", + "got_raw": "text.lower()", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "MarshalX/atproto", + "expected": "expected_signing_key", + "got": "expected_signing_key", + "got_raw": "expected_signing_key", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MarshalX/atproto", + "expected": "expected_pds", + "got": "expected_pds", + "got_raw": "expected_pds", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MarshalX/atproto", + "expected": "ValidationError)", + "got": "ValidationError)", + "got_raw": "ValidationError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MarshalX/atproto", + "expected": "DidNotFoundError)", + "got": "DidNotFoundError)", + "got_raw": "DidNotFoundError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MarshalX/atproto", + "expected": "expected_hostname", + "got": "expected_hostname", + "got_raw": "expected_hostname", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MarshalX/atproto", + "expected": "'application/json'", + "got": "'application/json'", + "got_raw": "'application/json'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MarshalX/atproto", + "expected": "valid_data['did']", + "got": "valid_data['did']", + "got_raw": "valid_data['did']", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MarshalX/atproto", + "expected": "client.request.get_headers()", + "got": "dm_client.headers", + "got_raw": "dm_client.headers", + "exact_match": false, + "code_bleu": 0.19362900077353815, + "edit_similarity": 0.6222222222222222 + }, + { + "repo": "MarshalX/atproto", + "expected": "'2023-07-20T10:17:40.298101'", + "got": "'2023-01-01T00:00:00Z'", + "got_raw": "'2023-01-01T00:00:00Z'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.44 + }, + { + "repo": "MarshalX/atproto", + "expected": "str(at_uri)", + "got": "str(at_uri)", + "got_raw": "str(at_uri)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MarshalX/atproto", + "expected": "expected_did_plc", + "got": "expected_did_plc", + "got_raw": "expected_did_plc", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MarshalX/atproto", + "expected": "car.root", + "got": "car.root", + "got_raw": "car.root", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MarshalX/atproto", + "expected": "'foo'", + "got": "'foo'", + "got_raw": "'foo'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MarshalX/atproto", + "expected": "expected_json_schema", + "got": "expected_json_schema", + "got_raw": "expected_json_schema", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MarshalX/atproto", + "expected": "blob_ref.ref.link", + "got": "blob_ref.to_ipld_json()['ref']['link']", + "got_raw": "blob_ref.to_ipld_json()['ref']['link']", + "exact_match": false, + "code_bleu": 0.14991106946711685, + "edit_similarity": 0.5818181818181818 + }, + { + "repo": "MarshalX/atproto", + "expected": "expected_purpose", + "got": "expected_purpose", + "got_raw": "expected_purpose", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MarshalX/atproto", + "expected": "expected_pubkey_multibase", + "got": "did_key2", + "got_raw": "did_key2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.30303030303030304 + }, + { + "repo": "MarshalX/atproto", + "expected": "expected_did", + "got": "expected_did", + "got_raw": "expected_did", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MarshalX/atproto", + "expected": "bytes_blob_ref.ref", + "got": "plain_cid", + "got_raw": "plain_cid", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.14814814814814814 + }, + { + "repo": "MarshalX/atproto", + "expected": "did_doc", + "got": "did_doc", + "got_raw": "did_doc", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "self.bag.messages())", + "got": "[\"email\"]", + "got_raw": "[\"email\"]", + "exact_match": false, + "code_bleu": 0.1424691910596736, + "edit_similarity": 0.20689655172413793 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "str", + "got": "str", + "got_raw": "str", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "location)", + "got": "location)", + "got_raw": "location)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "self.bag.empty())", + "got": "self.bag.has(\"email\"))", + "got_raw": "self.bag.has(\"email\"))", + "exact_match": false, + "code_bleu": 0.4671379777282001, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "bindings.keys())", + "got": "bindings)", + "got_raw": "bindings)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.72 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "\"value\")", + "got": "\"value\")", + "got_raw": "\"value\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "user.notifications.count())", + "got": "len(user.notifications))", + "got_raw": "len(user.notifications))", + "exact_match": false, + "code_bleu": 0.3738761305967048, + "edit_similarity": 0.7843137254901961 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "\"2\")", + "got": "\"2\")", + "got_raw": "\"2\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "count + 1", + "got": "count + 1", + "got_raw": "count + 1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "\"c\")", + "got": "\"c\")", + "got_raw": "\"c\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "None)", + "got": "None)", + "got_raw": "None)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "self.errors.messages())", + "got": "self.errors.get_messages(\"email\"))", + "got_raw": "self.errors.get_messages(\"email\"))", + "exact_match": false, + "code_bleu": 0.4671379777282001, + "edit_similarity": 0.8070175438596491 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "\"arg2\")", + "got": "\"arg2\")", + "got_raw": "\"arg2\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "validate.get(\"age\"))", + "got": "validate.errors)", + "got_raw": "validate.errors)", + "exact_match": false, + "code_bleu": 0.29036157037406557, + "edit_similarity": 0.6111111111111112 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "parse_human_time(\"2 hour\"))", + "got": "pendulum.datetime(2021, 1, 1, 12, 0, 0))", + "got_raw": "pendulum.datetime(2021, 1, 1, 12, 0, 0))", + "exact_match": false, + "code_bleu": 0.10216198665886358, + "edit_similarity": 0.417910447761194 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "SayHello)", + "got": "SayHello)", + "got_raw": "SayHello)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "request.accepts_json())", + "got": "request.accepts_json())", + "got_raw": "request.accepts_json())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "value", + "got": "value", + "got_raw": "value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "\"\")", + "got": "\"\")", + "got_raw": "\"\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MasoniteFramework/masonite", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MaxHalford/prince", + "expected": "[ \"alcohol_by_volume\", \"final_gravity\", \"international_bitterness_units\", \"standard_reference_method\", ]", + "got": "sorted(self.sup_cols)", + "got_raw": "sorted(self.sup_cols)", + "exact_match": false, + "code_bleu": 0.1114503479948016, + "edit_similarity": 0.144 + }, + { + "repo": "MaxHalford/prince", + "expected": "aligned_shapes.shape)", + "got": "aligned_shapes.shape)", + "got_raw": "aligned_shapes.shape)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MaxHalford/prince", + "expected": "P * 100)", + "got": "P)", + "got_raw": "P)", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.4 + }, + { + "repo": "MaxHalford/prince", + "expected": "P * 100)", + "got": "P)", + "got_raw": "P)", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.4 + }, + { + "repo": "MaxHalford/prince", + "expected": "np.abs(P))", + "got": "np.abs(P))", + "got_raw": "np.abs(P))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Mayitzin/ahrs", + "expected": "0.1)", + "got": "1.0)", + "got_raw": "1.0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "Mayitzin/ahrs", + "expected": "0.0)", + "got": "0.0)", + "got_raw": "0.0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Mayitzin/ahrs", + "expected": "1.0)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "Mayitzin/ahrs", + "expected": "self.q4.w)", + "got": "self.q4[0])", + "got_raw": "self.q4[0])", + "exact_match": false, + "code_bleu": 0.3655552228545123, + "edit_similarity": 0.7619047619047619 + }, + { + "repo": "Mayitzin/ahrs", + "expected": "3.0)", + "got": "2.0)", + "got_raw": "2.0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.75 + }, + { + "repo": "MerrimanInd/drawpyo", + "expected": "\"important\"", + "got": "\"important\"", + "got_raw": "\"important\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MerrimanInd/drawpyo", + "expected": "l", + "got": "l", + "got_raw": "l", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MerrimanInd/drawpyo", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MerrimanInd/drawpyo", + "expected": "p1", + "got": "p1", + "got_raw": "p1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MerrimanInd/drawpyo", + "expected": "(30, 40)", + "got": "(30, 40)", + "got_raw": "(30, 40)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MerrimanInd/drawpyo", + "expected": "20", + "got": "20", + "got_raw": "20", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MerrimanInd/drawpyo", + "expected": "obj_style", + "got": "obj_style", + "got_raw": "obj_style", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MerrimanInd/drawpyo", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MerrimanInd/drawpyo", + "expected": "50", + "got": "50", + "got_raw": "50", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MerrimanInd/drawpyo", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MerrimanInd/drawpyo", + "expected": "oy + 30", + "got": "oy + 30", + "got_raw": "oy + 30", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MerrimanInd/drawpyo", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MerrimanInd/drawpyo", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MerrimanInd/drawpyo", + "expected": "\"Hell0_w0rld\"", + "got": "\"Hello_World\"", + "got_raw": "\"Hello_World\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "MerrimanInd/drawpyo", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MerrimanInd/drawpyo", + "expected": "20", + "got": "20", + "got_raw": "20", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/model2vec", + "expected": "v", + "got": "v", + "got_raw": "v", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/model2vec", + "expected": "(2, 2)", + "got": "(2, 2)", + "got_raw": "(2, 2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/model2vec", + "expected": "[1]", + "got": "[1]", + "got_raw": "[1]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/model2vec", + "expected": "(2,)", + "got": "(2, 128)", + "got_raw": "(2, 128)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "MinishLab/model2vec", + "expected": "static_model2.base_model_name", + "got": "static_model2.base_model_name", + "got_raw": "static_model2.base_model_name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/model2vec", + "expected": "15_000", + "got": "15_000", + "got_raw": "15_000", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/model2vec", + "expected": "expected_vocab_size", + "got": "expected_vocab_size", + "got_raw": "expected_vocab_size", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "selected", + "got": "selected", + "got_raw": "selected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "0.2", + "got": "0.8", + "got_raw": "0.8", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "MinishLab/semhash", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MinishLab/semhash", + "expected": "0.4", + "got": "0.8", + "got_raw": "0.8", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "mol2", + "got": "mol1", + "got_raw": "mol1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "4", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "scorer(mol), abs=1e-3)", + "got": "0.0)", + "got_raw": "0.0)", + "exact_match": false, + "code_bleu": 0.097688175620322, + "edit_similarity": 0.07692307692307693 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "BIG_INT - 1", + "got": "BIG_INT", + "got_raw": "BIG_INT", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "[\"C\"]", + "got": "[\"C\"]", + "got_raw": "[\"C\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "\"CCO\"", + "got": "\"CCO\"", + "got_raw": "\"CCO\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "reaction.mol.smiles", + "got": "reaction.mol.smiles", + "got_raw": "reaction.mol.smiles", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "0", + "got": "0.0", + "got_raw": "0.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "StopIteration)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.24 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "[id(mol1), id_]", + "got": "[id(mol1), id(mol2)]", + "got_raw": "[id(mol1), id(mol2)]", + "exact_match": false, + "code_bleu": 0.5331675363405771, + "edit_similarity": 0.8 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "lines", + "got": "lines", + "got_raw": "lines", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "mol1.smiles", + "got": "mol1.smiles", + "got_raw": "mol1.smiles", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "priors2", + "got": "priors2", + "got_raw": "priors2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "tree.root.mol", + "got": "tree.root.mol", + "got_raw": "tree.root.mol", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MolecularAI/aizynthfinder", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "4", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "PULL", + "got": "PULL", + "got_raw": "PULL", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"1\"", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "Color.RED", + "got": "Color.RED", + "got_raw": "Color.RED", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "d2.date()", + "got": "d2.date()", + "got_raw": "d2.date()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "p.age", + "got": "p.age", + "got_raw": "p.age", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "[Human]", + "got": "[Human]", + "got_raw": "[Human]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"@\"", + "got": "\"user\"", + "got_raw": "\"user\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "2", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "doc2", + "got": "doc1", + "got_raw": "doc1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "\"me\"", + "got": "\"me\"", + "got_raw": "\"me\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MongoEngine/mongoengine", + "expected": "s2", + "got": "s2", + "got_raw": "s2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MultimediaTechLab/YOLO", + "expected": "list(expected_paths)", + "got": "expected_paths", + "got_raw": "expected_paths", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "MultimediaTechLab/YOLO", + "expected": "(2, 5)", + "got": "(2, 3)", + "got_raw": "(2, 3)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "MultimediaTechLab/YOLO", + "expected": "l_images.shape", + "got": "l_images.shape", + "got_raw": "l_images.shape", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MultimediaTechLab/YOLO", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MultimediaTechLab/YOLO", + "expected": "l_reverse_tensors.shape", + "got": "l_reverse_tensors.shape", + "got_raw": "l_reverse_tensors.shape", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "MultimediaTechLab/YOLO", + "expected": "(1, OUT_CHANNELS, 64, 64)", + "got": "torch.Size([1, OUT_CHANNELS, 64, 64])", + "got_raw": "torch.Size([1, OUT_CHANNELS, 64, 64])", + "exact_match": false, + "code_bleu": 0.5031747626530138, + "edit_similarity": 0.8064516129032258 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "\"500-\"", + "got": "\"RangePart(500, None)\"", + "got_raw": "\"RangePart(500, None)\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.35714285714285715 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "\"/\"", + "got": "\"/\"", + "got_raw": "\"/\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "\"a\"", + "got": "\"a\"", + "got_raw": "\"a\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "parsed", + "got": "parsed", + "got_raw": "parsed", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "utcnow()", + "got": "utcnow()", + "got_raw": "utcnow()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "headers", + "got": "headers", + "got_raw": "headers", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "\"\"", + "got": "scope[\"root_path\"]", + "got_raw": "scope[\"root_path\"]", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.2 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "b\"true\"", + "got": "b\"true\"", + "got_raw": "b\"true\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "foo", + "got": "foo", + "got_raw": "foo", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "\"/app\"", + "got": "\"/app\"", + "got_raw": "\"/app\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "\"file1\"", + "got": "\"file1\"", + "got_raw": "\"file1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Neoteroi/BlackSheep", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "NewFuture/DDNS", + "expected": "\"@\")", + "got": "\"@\")", + "got_raw": "\"@\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "NewFuture/DDNS", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "NewFuture/DDNS", + "expected": "\"\")", + "got": "\"\")", + "got_raw": "\"\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "NewFuture/DDNS", + "expected": "\"\")", + "got": "b\"\")", + "got_raw": "b\"\")", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "NewFuture/DDNS", + "expected": "\"\")", + "got": "\"\")", + "got_raw": "\"\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "NewFuture/DDNS", + "expected": "bool)", + "got": "bool)", + "got_raw": "bool)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "NewFuture/DDNS", + "expected": "\"www\")", + "got": "\"www\")", + "got_raw": "\"www\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "NewFuture/DDNS", + "expected": "int)", + "got": "int)", + "got_raw": "int)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "NewFuture/DDNS", + "expected": "result)", + "got": "result)", + "got_raw": "result)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "NewFuture/DDNS", + "expected": "\"cron\")", + "got": "\"cron\")", + "got_raw": "\"cron\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "NewFuture/DDNS", + "expected": "\"\")", + "got": "\"\")", + "got_raw": "\"\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "NewFuture/DDNS", + "expected": "\"0.5\")", + "got": "0.5)", + "got_raw": "0.5)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8 + }, + { + "repo": "NewFuture/DDNS", + "expected": "body)", + "got": "body)", + "got_raw": "body)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "NewFuture/DDNS", + "expected": "\"r\")", + "got": "\"r\", encoding=self.test_encoding)", + "got_raw": "\"r\", encoding=self.test_encoding)", + "exact_match": false, + "code_bleu": 0.17747405280050269, + "edit_similarity": 0.21621621621621623 + }, + { + "repo": "NewFuture/DDNS", + "expected": "\"GET\")", + "got": "self.id)", + "got_raw": "self.id)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "NewFuture/DDNS", + "expected": "3)", + "got": "3)", + "got_raw": "3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "NewFuture/DDNS", + "expected": "\"\u8054\u901a\")", + "got": "\"\u8054\u901a\")", + "got_raw": "\"\u8054\u901a\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "NewFuture/DDNS", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "NewFuture/DDNS", + "expected": "\"456\")", + "got": "\"456\")", + "got_raw": "\"456\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "NewFuture/DDNS", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "OrionStarAI/DeepV-Ki", + "expected": "\"TEST_001\"", + "got": "\"TEST_001\"", + "got_raw": "\"TEST_001\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "OrionStarAI/DeepV-Ki", + "expected": "data", + "got": "data", + "got_raw": "data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "OrionStarAI/DeepV-Ki", + "expected": "\"integration_test@example.com\"", + "got": "user_data[\"uid\"]", + "got_raw": "user_data[\"uid\"]", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.2608695652173913 + }, + { + "repo": "OrionStarAI/DeepV-Ki", + "expected": "\"user\"", + "got": "\"user\"", + "got_raw": "\"user\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "OrionStarAI/DeepV-Ki", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PedroBern/django-graphql-auth", + "expected": "executed[\"refreshToken\"])", + "got": "user.is_verified)", + "got_raw": "user.is_verified)", + "exact_match": false, + "code_bleu": 0.28744328957046744, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "PedroBern/django-graphql-auth", + "expected": "True)", + "got": "True)", + "got_raw": "True)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PedroBern/django-graphql-auth", + "expected": "False)", + "got": "False)", + "got_raw": "False)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PedroBern/django-graphql-auth", + "expected": "{\"verified\": False})", + "got": "{\"publicUser\": {\"verified\": False}})", + "got_raw": "{\"publicUser\": {\"verified\": False}})", + "exact_match": false, + "code_bleu": 0.5022573937283119, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "PedroBern/django-graphql-auth", + "expected": "True)", + "got": "True)", + "got_raw": "True)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PedroBern/django-graphql-auth", + "expected": "executed[\"errors\"])", + "got": "executed[\"errors\"][\"password\"])", + "got_raw": "executed[\"errors\"][\"password\"])", + "exact_match": false, + "code_bleu": 0.493938273711537, + "edit_similarity": 0.76 + }, + { + "repo": "PedroBern/django-graphql-auth", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PedroBern/django-graphql-auth", + "expected": "executed[\"errors\"][\"newPassword2\"])", + "got": "Messages.PASSWORDS_MISMATCH in executed[\"errors\"])", + "got_raw": "Messages.PASSWORDS_MISMATCH in executed[\"errors\"])", + "exact_match": false, + "code_bleu": 0.4347208719449914, + "edit_similarity": 0.4470588235294118 + }, + { + "repo": "PedroBern/django-graphql-auth", + "expected": "{ \"nonFieldErrors\": [ { \"code\": \"password_already_set\", \"message\": \"Password already set for account.\", } ] })", + "got": "Messages.ALREADY_SET_PASSWORD)", + "got_raw": "Messages.ALREADY_SET_PASSWORD)", + "exact_match": false, + "code_bleu": 0.08454214399131395, + "edit_similarity": 0.12857142857142856 + }, + { + "repo": "PedroBern/django-graphql-auth", + "expected": "True)", + "got": "True)", + "got_raw": "True)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PedroBern/django-graphql-auth", + "expected": "None)", + "got": "None)", + "got_raw": "None)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PedroBern/django-graphql-auth", + "expected": "Messages.UNAUTHENTICATED)", + "got": "Messages.UNAUTHENTICATED)", + "got_raw": "Messages.UNAUTHENTICATED)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "w[2])", + "got": "w[2])", + "got_raw": "w[2])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "11)", + "got": "22)", + "got_raw": "22)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "l)", + "got": "l)", + "got_raw": "l)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "err.OperationalError", + "got": "err.OperationalError", + "got_raw": "err.OperationalError", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "self.connection.literal(3.1415))", + "got": "self.db.literal(3.1415e0))", + "got_raw": "self.db.literal(3.1415e0))", + "exact_match": false, + "code_bleu": 0.3549481056010053, + "edit_similarity": 0.7586206896551724 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "(1045, \"Access denied\")", + "got": "(28000, \"Access denied\")", + "got_raw": "(28000, \"Access denied\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.851063829787234 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "orphans)", + "got": ".leak_test = False", + "got_raw": ".leak_test = False", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "r)", + "got": "r)", + "got_raw": "r)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "self.driver.Error)", + "got": "self.driver.Error)", + "got_raw": "self.driver.Error)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "(\"row1\", 0))", + "got": "(\"row1\", 0.01))", + "got_raw": "(\"row1\", 0.01))", + "exact_match": false, + "code_bleu": 0.488923022434901, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "cur.fetchall())", + "got": "list(cur))", + "got_raw": "list(cur))", + "exact_match": false, + "code_bleu": 0.34020690871988585, + "edit_similarity": 0.4 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "lowercase_latin1", + "got": "lowercase_latin1", + "got_raw": "lowercase_latin1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "ER.NO_SUCH_TABLE)", + "got": "ER.BAD_TABLE_ERROR)", + "got_raw": "ER.BAD_TABLE_ERROR)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "PyMySQL/PyMySQL", + "expected": "[(2,)])", + "got": "[(2,)])", + "got_raw": "[(2,)])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "3)", + "got": "3)", + "got_raw": "3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "4)", + "got": "10)", + "got_raw": "10)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "(-103, True))", + "got": "-103)", + "got_raw": "-103)", + "exact_match": false, + "code_bleu": 0.27383275896268816, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "pickup)", + "got": "pickup)", + "got_raw": "pickup)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "11 + 3 * 9)", + "got": "11 + 3 * 9)", + "got_raw": "11 + 3 * 9)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "x)", + "got": "x)", + "got_raw": "x)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "[5])", + "got": "10)", + "got_raw": "10)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "stats)", + "got": "stats)", + "got_raw": "stats)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "3)", + "got": "10)", + "got_raw": "10)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "0)", + "got": "num_neighbours)", + "got_raw": "num_neighbours)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "client)", + "got": "client)", + "got_raw": "client)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyVRP/PyVRP", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyWavelets/pywt", + "expected": "'haar')", + "got": "'haar')", + "got_raw": "'haar')", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyWavelets/pywt", + "expected": "(5, 3))", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.17909498863725698, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "PyWavelets/pywt", + "expected": "np.linspace(0, 5, x.size, endpoint=False))", + "got": "phi_d)", + "got_raw": "phi_d)", + "exact_match": false, + "code_bleu": 0.06320999598962011, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "PyWavelets/pywt", + "expected": "cD0)", + "got": "cD0)", + "got_raw": "cD0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyWavelets/pywt", + "expected": "None)", + "got": "0.5)", + "got_raw": "0.5)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "PyWavelets/pywt", + "expected": "np.array([5., 13.]))", + "got": "np.array([1.05066017179, 3.90190738382, 6.75315459585, 9.60440180788]))", + "got_raw": "np.array([1.05066017179, 3.90190738382, 6.75315459585, 9.60440180788]))", + "exact_match": false, + "code_bleu": 0.393755531055134, + "edit_similarity": 0.3956043956043956 + }, + { + "repo": "PyWavelets/pywt", + "expected": "v)", + "got": "v)", + "got_raw": "v)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyWavelets/pywt", + "expected": "[])", + "got": "X)", + "got_raw": "X)", + "exact_match": false, + "code_bleu": 0.35818997727451396, + "edit_similarity": 0.4 + }, + { + "repo": "PyWavelets/pywt", + "expected": "expected[i, :])", + "got": "expected[i])", + "got_raw": "expected[i])", + "exact_match": false, + "code_bleu": 0.4316250567652526, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "PyWavelets/pywt", + "expected": "c)", + "got": "c)", + "got_raw": "c)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyWavelets/pywt", + "expected": "[[0, 1]] * 2)", + "got": "pywt.threshold(data, 2, 'soft', 1))", + "got_raw": "pywt.threshold(data, 2, 'soft', 1))", + "exact_match": false, + "code_bleu": 0.14283632578659286, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "PyWavelets/pywt", + "expected": "res2)", + "got": "res2)", + "got_raw": "res2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "PyWavelets/pywt", + "expected": "pywt.WaveletPacket)", + "got": "pywt.WaveletPacket, data=x, wavelet='db1', mode='symmetric', axis=2)", + "got_raw": "pywt.WaveletPacket, data=x, wavelet='db1', mode='symmetric', axis=2)", + "exact_match": false, + "code_bleu": 0.12021577610863723, + "edit_similarity": 0.4367816091954023 + }, + { + "repo": "PyWavelets/pywt", + "expected": "x)", + "got": "x)", + "got_raw": "x)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Quantweb3-com/NexusTrader", + "expected": "balances", + "got": "balances", + "got_raw": "balances", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Quantweb3-com/NexusTrader", + "expected": "-1", + "got": "Decimal(\"-1\")", + "got_raw": "Decimal(\"-1\")", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "Quantweb3-com/NexusTrader", + "expected": "btc.total", + "got": "balances", + "got_raw": "balances", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "Quantweb3-com/NexusTrader", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "diff", + "got": "diff", + "got_raw": "diff", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "[on_next(340, 8)]", + "got": "[on_next(300, 7), on_next(300, 3), on_next(300, 4), on_next(300, 1), on_next(300, 8), on_next(300, 5), on_next(300, 6), on_next(300, 7), on_next(300, 13), on_next(300, 2), on_next(300, 9), on_next(300, 11), on_next(30", + "got_raw": "[on_next(300, 7), on_next(300, 3), on_next(300, 4), on_next(300, 1), on_next(300, 8), on_next(300, 5), on_next(300, 6), on_next(300, 7), on_next(300, 13), on_next(300, 2), on_next(300, 9), on_next(300, 11), on_next(30", + "exact_match": false, + "code_bleu": 0.03658682550555312, + "edit_similarity": 0.13675213675213677 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "[8.0]", + "got": "[8.0]", + "got_raw": "[8.0]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "[on_completed(225)]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "[]", + "got": "r_msgs", + "got_raw": "r_msgs", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "[1, 2, 3]", + "got": "[1, 2, 3]", + "got_raw": "[1, 2, 3]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "i[0])", + "got": "i[0])", + "got_raw": "i[0])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ReactiveX/RxPY", + "expected": "e", + "got": "Exception(\"error\")", + "got_raw": "Exception(\"error\")", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.10526315789473684 + }, + { + "repo": "SALib/SALib", + "expected": "[0.13] * 5)", + "got": "[0.0, 0.0, 0.0, 0.0, 0.0])", + "got_raw": "[0.0, 0.0, 0.0, 0.0, 0.0])", + "exact_match": false, + "code_bleu": 0.1235622127262679, + "edit_similarity": 0.32432432432432434 + }, + { + "repo": "SALib/SALib", + "expected": "[0.00, 0.25, 0.00])", + "got": "[0.00, 0.00, 0.00])", + "got_raw": "[0.00, 0.00, 0.00])", + "exact_match": false, + "code_bleu": 0.5969491792019646, + "edit_similarity": 0.8947368421052632 + }, + { + "repo": "SALib/SALib", + "expected": "3)", + "got": "3)", + "got_raw": "3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SALib/SALib", + "expected": "[0.00, 0.25, 0.00])", + "got": "[0.00, 0.00, 0.00])", + "got_raw": "[0.00, 0.00, 0.00])", + "exact_match": false, + "code_bleu": 0.5969491792019646, + "edit_similarity": 0.8947368421052632 + }, + { + "repo": "SALib/SALib", + "expected": "[0.00, 0.25, 0.00])", + "got": "[0.00, 0.00, 0.00])", + "got_raw": "[0.00, 0.00, 0.00])", + "exact_match": false, + "code_bleu": 0.5969491792019646, + "edit_similarity": 0.8947368421052632 + }, + { + "repo": "SALib/SALib", + "expected": "psp.analysis[col])", + "got": "psp.analysis[col])", + "got_raw": "psp.analysis[col])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SALib/SALib", + "expected": "[N * (D + 2), D])", + "got": "(N, D))", + "got_raw": "(N, D))", + "exact_match": false, + "code_bleu": 0.17965205598154213, + "edit_similarity": 0.4166666666666667 + }, + { + "repo": "SALib/SALib", + "expected": "Si[\"ME\"])", + "got": "Si)", + "got_raw": "Si)", + "exact_match": false, + "code_bleu": 0.23784142300054423, + "edit_similarity": 0.5 + }, + { + "repo": "SALib/SALib", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SALib/SALib", + "expected": "desired_mu)", + "got": "desired_mu)", + "got_raw": "desired_mu)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SAML-Toolkits/python-saml", + "expected": "metadata_without_descriptors)", + "got": "metadata_without_descriptors)", + "got_raw": "metadata_without_descriptors)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SAML-Toolkits/python-saml", + "expected": "logout_response.get_error())", + "got": "logout_response.get_error())", + "got_raw": "logout_response.get_error())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SAML-Toolkits/python-saml", + "expected": "nameid_data)", + "got": "nameid_data)", + "got_raw": "nameid_data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SAML-Toolkits/python-saml", + "expected": "name_id_data_4)", + "got": "name_id_data_4)", + "got_raw": "name_id_data_4)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SAML-Toolkits/python-saml", + "expected": "metadata)", + "got": "metadata)", + "got_raw": "metadata)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SAML-Toolkits/python-saml", + "expected": "settings5)", + "got": "settings5)", + "got_raw": "settings5)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SAML-Toolkits/python-saml", + "expected": "nameid_data_3)", + "got": "nameid_data_3)", + "got_raw": "nameid_data_3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SAML-Toolkits/python-saml", + "expected": "data)", + "got": "data)", + "got_raw": "data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SAML-Toolkits/python3-saml", + "expected": "response_4.is_valid(request_data))", + "got": "response_4.is_valid(request_data))", + "got_raw": "response_4.is_valid(request_data))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SAML-Toolkits/python3-saml", + "expected": "target_url2)", + "got": "url)", + "got_raw": "url)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "SAML-Toolkits/python3-saml", + "expected": "response_2.get_status())", + "got": "response_2.get_status())", + "got_raw": "response_2.get_status())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SAML-Toolkits/python3-saml", + "expected": "nameid)", + "got": "nameid)", + "got_raw": "nameid)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SAML-Toolkits/python3-saml", + "expected": "name_id_data)", + "got": "name_id_data)", + "got_raw": "name_id_data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SAML-Toolkits/python3-saml", + "expected": "response.is_valid(request_data))", + "got": "response.is_invalid_xml())", + "got_raw": "response.is_invalid_xml())", + "exact_match": false, + "code_bleu": 0.32794537853815264, + "edit_similarity": 0.6896551724137931 + }, + { + "repo": "SAML-Toolkits/python3-saml", + "expected": "settings3)", + "got": "settings3)", + "got_raw": "settings3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SAML-Toolkits/python3-saml", + "expected": "Exception)", + "got": "Exception)", + "got_raw": "Exception)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SAML-Toolkits/python3-saml", + "expected": "inflated_3)", + "got": "inflated_3)", + "got_raw": "inflated_3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-agent/SWE-ReX", + "expected": "\"\\n\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "SWE-agent/SWE-ReX", + "expected": "RuntimeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "SWE-agent/SWE-ReX", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-agent/SWE-ReX", + "expected": "DeploymentNotStartedError)", + "got": "DeploymentNotStartedError)", + "got_raw": "DeploymentNotStartedError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-agent/SWE-ReX", + "expected": "\"podman\"", + "got": "\"podman\"", + "got_raw": "\"podman\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "\"FAILED\"", + "got": "\"FAILED\"", + "got_raw": "\"FAILED\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "result.rewrite", + "got": "result", + "got_raw": "result", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.6 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "\"SomeMethod\"", + "got": "\"Utf8\"", + "got_raw": "\"Utf8\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "4", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "names", + "got": "names", + "got_raw": "names", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "src", + "got": "src", + "got_raw": "src", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "rsa_key", + "got": "rsa_key", + "got_raw": "rsa_key", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SWE-bench/SWE-smith", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Significant-Gravitas/Auto-GPT-Plugins", + "expected": "MOCK_FROM", + "got": "MOCK_FROM", + "got_raw": "MOCK_FROM", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Significant-Gravitas/Auto-GPT-Plugins", + "expected": "str)", + "got": "str)", + "got_raw": "str)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Significant-Gravitas/Auto-GPT-Plugins", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Significant-Gravitas/Auto-GPT-Plugins", + "expected": "401)", + "got": "403)", + "got_raw": "403)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.75 + }, + { + "repo": "Significant-Gravitas/Auto-GPT-Plugins", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Significant-Gravitas/Auto-GPT-Plugins", + "expected": "401)", + "got": "403)", + "got_raw": "403)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.75 + }, + { + "repo": "SmileyChris/django-countries", + "expected": "5)", + "got": "5)", + "got_raw": "5)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SmileyChris/django-countries", + "expected": "\"\")", + "got": "None)", + "got_raw": "None)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.25 + }, + { + "repo": "SmileyChris/django-countries", + "expected": "\"\ud83c\uddf3\ud83c\uddff\")", + "got": "\"nz\")", + "got_raw": "\"nz\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6 + }, + { + "repo": "SmileyChris/django-countries", + "expected": "900)", + "got": "900)", + "got_raw": "900)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SmileyChris/django-countries", + "expected": "200)", + "got": "200)", + "got_raw": "200)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SmileyChris/django-countries", + "expected": "\"\")", + "got": "\"US\")", + "got_raw": "\"US\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.75 + }, + { + "repo": "SmileyChris/django-countries", + "expected": "list(models.Payment.objects.all()))", + "got": "models.Payment.objects.all())", + "got_raw": "models.Payment.objects.all())", + "exact_match": false, + "code_bleu": 0.7692307692307693, + "edit_similarity": 0.90625 + }, + { + "repo": "SmileyChris/django-countries", + "expected": "\"NZ\")", + "got": "\"NZ\")", + "got_raw": "\"NZ\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SmileyChris/django-countries", + "expected": "200)", + "got": "10)", + "got_raw": "10)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "SmileyChris/django-countries", + "expected": "\"CA\")", + "got": "\"FR\")", + "got_raw": "\"FR\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6 + }, + { + "repo": "SmileyChris/django-countries", + "expected": "result)", + "got": "result)", + "got_raw": "result)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "HMIP_DOMAIN", + "got": "HMIP_DOMAIN", + "got_raw": "HMIP_DOMAIN", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "\"press_short\"", + "got": "\"press_short\"", + "got_raw": "\"press_short\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "optional_settings", + "got": "optional_settings", + "got_raw": "optional_settings", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "128", + "got": "128", + "got_raw": "128", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "25", + "got": "25", + "got_raw": "25", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "state.attributes", + "got": "state.attributes", + "got_raw": "state.attributes", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "\"0.0\"", + "got": "\"0.0\"", + "got_raw": "\"0.0\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "24 * 60", + "got": "1440", + "got_raw": "1440", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "call_kwargs", + "got": "call_kwargs", + "got_raw": "call_kwargs", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "\"homematicip_local\"", + "got": "\"homematicip_local\"", + "got_raw": "\"homematicip_local\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "\"VCU0000001\"", + "got": "\"VCU0000001\"", + "got_raw": "\"VCU0000001\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "\"VCU2128127:1\"", + "got": "\"VCU2128127\"", + "got_raw": "\"VCU2128127\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "False", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "DeviceTriggerEventType.DEVICE_ERROR.value", + "got": "DeviceTriggerEventType.DEVICE_ERROR.value", + "got_raw": "DeviceTriggerEventType.DEVICE_ERROR.value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "\"evt_type\"", + "got": "config[dt.event_trigger.CONF_EVENT_TYPE]", + "got_raw": "config[dt.event_trigger.CONF_EVENT_TYPE]", + "exact_match": false, + "code_bleu": 0.13485111859503685, + "edit_similarity": 0.24 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "\"intf123\"", + "got": "\"intf123\"", + "got_raw": "\"intf123\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SukramJ/homematicip_local", + "expected": "{\"MONDAY\": [{\"start\": \"06:00\", \"temp\": 21.0}]}", + "got": "{\"MONDAY\": [{\"start\": \"06:00\", \"temp\": 21.0}]}", + "got_raw": "{\"MONDAY\": [{\"start\": \"06:00\", \"temp\": 21.0}]}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SuperClaude-Org/SuperClaude_Framework", + "expected": "recommendation", + "got": "recommendation", + "got_raw": "recommendation", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SuperClaude-Org/SuperClaude_Framework", + "expected": "message", + "got": "message", + "got_raw": "message", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SuperClaude-Org/SuperClaude_Framework", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "SuperClaude-Org/SuperClaude_Framework", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "8", + "got": "8", + "got_raw": "8", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "task", + "got": "task", + "got_raw": "task", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "\"id1\"", + "got": "ids[0]", + "got_raw": "ids[0]", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "chat", + "got": "chat", + "got_raw": "chat", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "\"\")", + "got": "'')", + "got_raw": "'')", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "stats[\"median\"]", + "got": "stats[\"median\"]", + "got_raw": "stats[\"median\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "Upsonic/Upsonic", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "XKNX/xknx", + "expected": "None", + "got": "False", + "got_raw": "False", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "XKNX/xknx", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "XKNX/xknx", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "XKNX/xknx", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "XKNX/xknx", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "XKNX/xknx", + "expected": "DPTArray(0xFF)", + "got": "DPTArray(255)", + "got_raw": "DPTArray(255)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7407407407407407 + }, + { + "repo": "XKNX/xknx", + "expected": "raw", + "got": "raw", + "got_raw": "raw", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "XKNX/xknx", + "expected": "value", + "got": "value", + "got_raw": "value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "XKNX/xknx", + "expected": "raw[8])", + "got": "0x21)", + "got_raw": "0x21)", + "exact_match": false, + "code_bleu": 0.2149139863647084, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "XKNX/xknx", + "expected": "data", + "got": "data", + "got_raw": "data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "XKNX/xknx", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "XKNX/xknx", + "expected": "raw", + "got": "raw", + "got_raw": "raw", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "XKNX/xknx", + "expected": "DPTArray(())", + "got": "DPTArray(())", + "got_raw": "DPTArray(())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "XKNX/xknx", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "XKNX/xknx", + "expected": "raw", + "got": "raw", + "got_raw": "raw", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "XKNX/xknx", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "XKNX/xknx", + "expected": "msg", + "got": "msg", + "got_raw": "msg", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "XKNX/xknx", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "XKNX/xknx", + "expected": "raw", + "got": "raw", + "got_raw": "raw", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "XKNX/xknx", + "expected": "\"Pa\"", + "got": "\"Pa\"", + "got_raw": "\"Pa\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agittins/bermuda", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agittins/bermuda", + "expected": "ICON_DEFAULT_AREA", + "got": "ICON_DEFAULT_AREA", + "got_raw": "ICON_DEFAULT_AREA", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agittins/bermuda", + "expected": "mock_scanner_device", + "got": "mock_scanner_device", + "got_raw": "mock_scanner_device", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agittins/bermuda", + "expected": "123.45", + "got": "123.45", + "got_raw": "123.45", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agittins/bermuda", + "expected": "123.45", + "got": "123.45", + "got_raw": "123.45", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agittins/bermuda", + "expected": "ICON_DEFAULT_FLOOR", + "got": "ICON_DEFAULT_FLOOR", + "got_raw": "ICON_DEFAULT_FLOOR", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agittins/bermuda", + "expected": "-20", + "got": "-20", + "got_raw": "-20", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/apscheduler", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/apscheduler", + "expected": "\"Cannot create a reference to a nested function\"", + "got": "\"nested is not a callable\"", + "got_raw": "\"nested is not a callable\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2972972972972973 + }, + { + "repo": "agronholm/apscheduler", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/apscheduler", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/apscheduler", + "expected": "MaxIterationsReached)", + "got": "MaxIterationsReached)", + "got_raw": "MaxIterationsReached)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/apscheduler", + "expected": "pkg2.varname", + "got": "pkg2.varname", + "got_raw": "pkg2.varname", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/apscheduler", + "expected": "correct_next_date", + "got": "correct_next_date", + "got_raw": "correct_next_date", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "agronholm/apscheduler", + "expected": "timestamp", + "got": "timestamp", + "got_raw": "timestamp", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aimagelab/mammoth", + "expected": "'2000'", + "got": "128", + "got_raw": "128", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "aimagelab/mammoth", + "expected": "'adam'", + "got": "'Adam'", + "got_raw": "'Adam'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "aimagelab/mammoth", + "expected": "'weak'", + "got": "'weak'", + "got_raw": "'weak'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aimagelab/mammoth", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aimagelab/mammoth", + "expected": "'1'", + "got": "'1'", + "got_raw": "'1'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "res2", + "got": "res2", + "got_raw": "res2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "3024", + "got": "1202", + "got_raw": "1202", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "str(w[-1].message)", + "got": "str(w[0].message)", + "got_raw": "str(w[0].message)", + "exact_match": false, + "code_bleu": 0.5922333058286215, + "edit_similarity": 0.9142857142857143 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "InterfaceError)", + "got": "NotSupportedError)", + "got_raw": "NotSupportedError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.48484848484848486 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "([('foo', 'bar')], [('bar', 'bat')]))", + "got": "[('a', 'b'), ('c', 'd')])", + "got_raw": "[('a', 'b'), ('c', 'd')])", + "exact_match": false, + "code_bleu": 0.17706805847543933, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "e.args[0]", + "got": "e.errno", + "got_raw": "e.errno", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.375 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "ProgrammingError)", + "got": "InterfaceError)", + "got_raw": "InterfaceError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4375 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "pool.size", + "got": "len(pool._connections)", + "got_raw": "len(pool._connections)", + "exact_match": false, + "code_bleu": 0.2907153684841096, + "edit_similarity": 0.3870967741935484 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "row.id", + "got": "row[1]", + "got_raw": "row[1]", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.5 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "22", + "got": "22", + "got_raw": "22", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "(1,)", + "got": "(1,)", + "got_raw": "(1,)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aio-libs/aiomysql", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aiogram/aiogram", + "expected": "str(event)", + "got": "str(event)", + "got_raw": "str(event)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aiogram/aiogram", + "expected": "value", + "got": "value", + "got_raw": "value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aiogram/aiogram", + "expected": "\"uk\"", + "got": "\"uk\"", + "got_raw": "\"uk\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aiogram/aiogram", + "expected": "\"test\"", + "got": "\"my\"", + "got_raw": "\"my\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "aiogram/aiogram", + "expected": "\"true\"", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6 + }, + { + "repo": "aiogram/aiogram", + "expected": "prepare_result.result", + "got": "prepare_result.result", + "got_raw": "prepare_result.result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aiogram/aiogram", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aiogram/aiogram", + "expected": "generator.bar", + "got": "generator.bar", + "got_raw": "generator.bar", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aiogram/aiogram", + "expected": "prepare_result.result", + "got": "prepare_result.result", + "got_raw": "prepare_result.result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aiogram/aiogram", + "expected": "json.dumps", + "got": "json.dumps", + "got_raw": "json.dumps", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aiogram/aiogram", + "expected": "\"baz\"", + "got": "await storage.set_data(key=storage_key, data={\"foo\": \"bar\"})", + "got_raw": "await storage.set_data(key=storage_key, data={\"foo\": \"bar\"})", + "exact_match": false, + "code_bleu": 0.06074588070876682, + "edit_similarity": 0.12307692307692308 + }, + { + "repo": "aiogram/aiogram", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aiogram/aiogram", + "expected": "event.user_chat_id", + "got": "event.chat.id", + "got_raw": "event.chat.id", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.7741935483870968 + }, + { + "repo": "aiogram/aiogram", + "expected": "id(markup)", + "got": "id(markup)", + "got_raw": "id(markup)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aiogram/aiogram", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "aiogram/aiogram", + "expected": "\"or\"", + "got": "\"!\"", + "got_raw": "\"!\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "aiogram/aiogram", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aiogram/aiogram", + "expected": "event", + "got": "event", + "got_raw": "event", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aiogram/aiogram", + "expected": "None", + "got": "UNSET_PARSE_MODE", + "got_raw": "UNSET_PARSE_MODE", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.1 + }, + { + "repo": "aiogram/aiogram", + "expected": "node", + "got": "node", + "got_raw": "node", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alexmojaki/futurecoder", + "expected": "\"Settings\"", + "got": "\"Settings\"", + "got_raw": "\"Settings\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alexmojaki/futurecoder", + "expected": "8)", + "got": "8)", + "got_raw": "8)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alexmojaki/futurecoder", + "expected": "parsons", + "got": "parsons", + "got_raw": "parsons", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alexmojaki/futurecoder", + "expected": "\"step_name_here\"", + "got": "\"Step\"", + "got_raw": "\"Step\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.45454545454545453 + }, + { + "repo": "alexmojaki/futurecoder", + "expected": "not parsons", + "got": "parsons", + "got_raw": "parsons", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "alexmojaki/futurecoder", + "expected": "vc2)", + "got": "vc2)", + "got_raw": "vc2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alexzhang13/rlm", + "expected": "\"tracked\"", + "got": "\"tracked\"", + "got_raw": "\"tracked\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alexzhang13/rlm", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alexzhang13/rlm", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alexzhang13/rlm", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alexzhang13/rlm", + "expected": "6", + "got": "6", + "got_raw": "6", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alexzhang13/rlm", + "expected": "captured_child_params", + "got": "captured_child_params", + "got_raw": "captured_child_params", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alexzhang13/rlm", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alexzhang13/rlm", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alexzhang13/rlm", + "expected": "out.stdout", + "got": "out", + "got_raw": "out", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "alexzhang13/rlm", + "expected": "second_env", + "got": "second_env", + "got_raw": "second_env", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alexzhang13/rlm", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alexzhang13/rlm", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alexzhang13/rlm", + "expected": "\"4\"", + "got": "\"FINAL(4)\"", + "got_raw": "\"FINAL(4)\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "response.status_code", + "got": "response.status_code", + "got_raw": "response.status_code", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "MemoryStorage", + "got": "MemoryStorage", + "got_raw": "MemoryStorage", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "b\"tres\"", + "got": "b\"tres\"", + "got_raw": "b\"tres\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "alisaifee/flask-limiter", + "expected": "result.output", + "got": "result.output", + "got_raw": "result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "True", + "got": "get_all_users", + "got_raw": "get_all_users", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "(annotation,)", + "got": "[annotation]", + "got_raw": "[annotation]", + "exact_match": false, + "code_bleu": 0.3388507513536918, + "edit_similarity": 0.8 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "[('world', False)]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.16988673071467442, + "edit_similarity": 0.2 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "[('hello', True)]", + "got": "['hello']", + "got_raw": "['hello']", + "exact_match": false, + "code_bleu": 0.23026498539075899, + "edit_similarity": 0.6923076923076923 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "ImportError)", + "got": "AttributeError)", + "got_raw": "AttributeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5185185185185185 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "\"a\"", + "got": "\"aa\"", + "got_raw": "\"aa\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "TypeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "[\"foo1\", \"foo2\"]", + "got": "[\"foo1\", \"foo2\"]", + "got_raw": "[\"foo1\", \"foo2\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "address_def['required']", + "got": "address_def['required']", + "got_raw": "address_def['required']", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "['b1', 'b2']", + "got": "['a']", + "got_raw": "['a']", + "exact_match": false, + "code_bleu": 0.29999999999999993, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "User", + "got": "User", + "got_raw": "User", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "allmonday/pydantic-resolve", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "amerkurev/doku", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "amerkurev/doku", + "expected": "'DU scanner stopped.')", + "got": "'Scan completed.')", + "got_raw": "'Scan completed.')", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6 + }, + { + "repo": "amerkurev/doku", + "expected": "1)", + "got": "10)", + "got_raw": "10)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8 + }, + { + "repo": "amerkurev/doku", + "expected": "[ call('/host/path', obj, ANY), call('/host/path', obj, ANY), call('/host/path', obj, ANY), ])", + "got": "_kvstore_set.return_value", + "got_raw": "_kvstore_set.return_value", + "exact_match": false, + "code_bleu": 0.03799178428257963, + "edit_similarity": 0.08403361344537816 + }, + { + "repo": "amerkurev/doku", + "expected": "LOGGER_NAME)", + "got": "LOGGER_NAME)", + "got_raw": "LOGGER_NAME)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "amerkurev/doku", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "amerkurev/doku", + "expected": "Path('/dest/subdir/file.txt')", + "got": "'/dest/subdir/file.txt'", + "got_raw": "'/dest/subdir/file.txt'", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.8846153846153846 + }, + { + "repo": "amerkurev/doku", + "expected": "[ call('image', images, ANY), call('container', DockerContainerList.model_validate(df_data['Containers']), ANY), call('volume', volumes, ANY), call('build_cache', DockerBuildCacheList.model_validate(df_data['BuildCache']), ANY), call('root_mount', docker_mount, ANY), ])", + "got": "kvstore_set.return_value", + "got_raw": "kvstore_set.return_value", + "exact_match": false, + "code_bleu": 0.021178171959605738, + "edit_similarity": 0.02040816326530612 + }, + { + "repo": "amerkurev/doku", + "expected": "'main:app'", + "got": "'main'", + "got_raw": "'main'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "amosjyng/langchain-visualizer", + "expected": "\"ell\"", + "got": "\" \"", + "got_raw": "\" \"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "amosjyng/langchain-visualizer", + "expected": "results[-1]", + "got": "results[2]", + "got_raw": "results[2]", + "exact_match": false, + "code_bleu": 0.4065061985236918, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "amosjyng/langchain-visualizer", + "expected": "( \"hello \", FValue(source=\"foo\", value=\"world\", formatted=\"world\"), )", + "got": "[langchain_visualizer.Node(\"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello", + "got_raw": "[langchain_visualizer.Node(\"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello {foo}\", \"hello", + "exact_match": false, + "code_bleu": 0.02872319364304312, + "edit_similarity": 0.11255411255411256 + }, + { + "repo": "amosjyng/langchain-visualizer", + "expected": "result.strip()", + "got": "result", + "got_raw": "result", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.6 + }, + { + "repo": "ansible-community/ansible-bender", + "expected": "logs", + "got": "logs", + "got_raw": "logs", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ansible-community/ansible-bender", + "expected": "\"leonardo\"", + "got": "\"1000\"", + "got_raw": "\"1000\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.25 + }, + { + "repo": "ansible-community/ansible-bender", + "expected": "\"ls\"", + "got": "[\"/bin/bash\"]", + "got_raw": "[\"/bin/bash\"]", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "ansible-community/ansible-bender", + "expected": "\"quay.io/biocontainers/python:3\"", + "got": "\"ubuntu:18.04\"", + "got_raw": "\"ubuntu:18.04\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.30434782608695654 + }, + { + "repo": "ansible-community/ansible-bender", + "expected": "4", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "ansible-community/ansible-bender", + "expected": "\"challet\"", + "got": "\"quay.io/biocontainers/python:3\"", + "got_raw": "\"quay.io/biocontainers/python:3\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.1951219512195122 + }, + { + "repo": "ansible-community/ansible-bender", + "expected": "\"y\"", + "got": "\"y\"", + "got_raw": "\"y\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ansible-community/ansible-bender", + "expected": "0", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "ansible-community/ansible-bender", + "expected": "cmd_e", + "got": "cmd_e", + "got_raw": "cmd_e", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "snapshot( { \"anyOf\": [ {\"type\": \"string\"}, { \"type\": \"integer\", \"description\": \"{minimum: 1}\", }, ] } )", + "got": "snapshot()", + "got_raw": "snapshot()", + "exact_match": false, + "code_bleu": 0.07372420433610836, + "edit_similarity": 0.17699115044247787 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "obj2", + "got": "obj2", + "got_raw": "obj2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "custom_schema", + "got": "custom_schema", + "got_raw": "custom_schema", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "snapshot({\"type\": \"string\"})", + "got": "snapshot()", + "got_raw": "snapshot()", + "exact_match": false, + "code_bleu": 0.23960366409235218, + "edit_similarity": 0.5263157894736842 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "snapshot({\"type\": \"null\"})", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "\"json_schema\"", + "got": "\"json_schema\"", + "got_raw": "\"json_schema\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "{\"prop\": None}", + "got": "DateDictWithRequiredAlias(required_prop=None)", + "got_raw": "DateDictWithRequiredAlias(required_prop=None)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.2711864406779661 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "\"in[]=foo&in[]=bar\"", + "got": "\"in[]=foo&in[]=bar\"", + "got_raw": "\"in[]=foo&in[]=bar\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "\"8\"", + "got": "\"8\"", + "got_raw": "\"8\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "type(None)", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "checking_client.beta.messages.tool_runner)", + "got": "\"get_weather\")", + "got_raw": "\"get_weather\")", + "exact_match": false, + "code_bleu": 0.13432124147794275, + "edit_similarity": 0.25 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "\"a[b]=c\"", + "got": "\"a[b]=c\"", + "got_raw": "\"a[b]=c\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "snapshot( { \"type\": \"string\", \"description\": \"\"\"\\ User email {default: user@example.com}\\ \"\"\", \"format\": \"email\", } )", + "got": "snapshot()", + "got_raw": "snapshot()", + "exact_match": false, + "code_bleu": 0.10088575330204301, + "edit_similarity": 0.15748031496062992 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "{\"foo\": \"bar\"}", + "got": "{\"foo\": \"bar\"}", + "got_raw": "{\"foo\": \"bar\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "tracked_types", + "got": "tracked_types", + "got_raw": "tracked_types", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "profiles[0][\"region\"]", + "got": "\"us-east-2\"", + "got_raw": "\"us-east-2\"", + "exact_match": false, + "code_bleu": 0.07142857142857142, + "edit_similarity": 0.1875 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "bytes", + "got": "ConcreteBaseAPIResponse", + "got_raw": "ConcreteBaseAPIResponse", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.21428571428571427 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "{\"foo\": \"bar\"}", + "got": "{\"foo\": \"bar\"}", + "got_raw": "{\"foo\": \"bar\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "anthropics/anthropic-sdk-python", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apiflask/apiflask", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apiflask/apiflask", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apiflask/apiflask", + "expected": "204", + "got": "204", + "got_raw": "204", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apiflask/apiflask", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apiflask/apiflask", + "expected": "100", + "got": "100", + "got_raw": "100", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apiflask/apiflask", + "expected": "result.output", + "got": "rv.data.decode()", + "got_raw": "rv.data.decode()", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.20689655172413793 + }, + { + "repo": "apiflask/apiflask", + "expected": "servers", + "got": "servers", + "got_raw": "servers", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apiflask/apiflask", + "expected": "'tags'", + "got": "'tags'", + "got_raw": "'tags'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apiflask/apiflask", + "expected": "app.info['description']", + "got": "'My API'", + "got_raw": "'My API'", + "exact_match": false, + "code_bleu": 0.08333333333333333, + "edit_similarity": 0.12903225806451613 + }, + { + "repo": "apiflask/apiflask", + "expected": "'foo'", + "got": "'foo'", + "got_raw": "'foo'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apiflask/apiflask", + "expected": "123", + "got": "123", + "got_raw": "123", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apiflask/apiflask", + "expected": "data", + "got": "data", + "got_raw": "data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apiflask/apiflask", + "expected": "'Unknown'", + "got": "'1234'", + "got_raw": "'1234'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "apiflask/apiflask", + "expected": "{}", + "got": "rv.json['paths']['/bar']['get']['responses']['200']['content']['image/png']", + "got_raw": "rv.json['paths']['/bar']['get']['responses']['200']['content']['image/png']", + "exact_match": false, + "code_bleu": 0.042601467364417965, + "edit_similarity": 0.0 + }, + { + "repo": "apiflask/apiflask", + "expected": "content", + "got": "content", + "got_raw": "content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apiflask/apiflask", + "expected": "201", + "got": "201", + "got_raw": "201", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apiflask/apiflask", + "expected": "['Parent']", + "got": "['Parent']", + "got_raw": "['Parent']", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apiflask/apiflask", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apiflask/apiflask", + "expected": "TypeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "apiflask/apiflask", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apragacz/django-rest-registration", + "expected": "email_change.new_value", + "got": "email_change.new_value", + "got_raw": "email_change.new_value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apragacz/django-rest-registration", + "expected": "response)", + "got": "response)", + "got_raw": "response)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apragacz/django-rest-registration", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apragacz/django-rest-registration", + "expected": "302", + "got": "302", + "got_raw": "302", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apragacz/django-rest-registration", + "expected": "[JWTAuthentication]", + "got": "(JWTAuthentication,)", + "got_raw": "(JWTAuthentication,)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.8717948717948718 + }, + { + "repo": "apragacz/django-rest-registration", + "expected": "timer)", + "got": "timer)", + "got_raw": "timer)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apragacz/django-rest-registration", + "expected": "\"Knuth\"", + "got": "\"Knuth\"", + "got_raw": "\"Knuth\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apragacz/django-rest-registration", + "expected": "url)", + "got": "_url = build_custom_verification_url(settings_with_register_verification.signer)", + "got_raw": "_url = build_custom_verification_url(settings_with_register_verification.signer)", + "exact_match": false, + "code_bleu": 0.16036590969929357, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "apragacz/django-rest-registration", + "expected": "[ ErrorCode.LOGIN_FIELDS_NOT_UNIQUE, ])", + "got": "[ErrorCode.LOGIN_FIELD_NOT_UNIQUE])", + "got_raw": "[ErrorCode.LOGIN_FIELD_NOT_UNIQUE])", + "exact_match": false, + "code_bleu": 0.41907687637277224, + "edit_similarity": 0.9459459459459459 + }, + { + "repo": "apragacz/django-rest-registration", + "expected": "new_email", + "got": "new_email", + "got_raw": "new_email", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "apragacz/django-rest-registration", + "expected": "expected_path", + "got": "expected_path", + "got_raw": "expected_path", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "mat_constrained_expected)", + "got": "mat_constrained_expected)", + "got_raw": "mat_constrained_expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "range_m_expected", + "got": "range_m_expected", + "got_raw": "range_m_expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "[93269500]", + "got": "[93269500]", + "got_raw": "[93269500]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "(4, 4, 3)", + "got": "(bev_img.shape[0], bev_img.shape[1], 3)", + "got_raw": "(bev_img.shape[0], bev_img.shape[1], 3)", + "exact_match": false, + "code_bleu": 0.1487964117124549, + "edit_similarity": 0.2916666666666667 + }, + { + "repo": "argoverse/av2-api", + "expected": "track_uuid_expected", + "got": "track_uuid_expected", + "got_raw": "track_uuid_expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "timestamp_ns_expected", + "got": "timestamp_ns_expected", + "got_raw": "timestamp_ns_expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "np.dtype(np.uint8)", + "got": "np.uint8", + "got_raw": "np.uint8", + "exact_match": false, + "code_bleu": 0.31533615572014295, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "argoverse/av2-api", + "expected": "lidar_tensor[:, 1:3])", + "got": "lidar.as_tensor())", + "got_raw": "lidar.as_tensor())", + "exact_match": false, + "code_bleu": 0.14726394089438588, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "argoverse/av2-api", + "expected": "4", + "got": "drivable_area.area_boundary[0].x", + "got_raw": "drivable_area.area_boundary[0].x", + "exact_match": false, + "code_bleu": 0.13485111859503685, + "edit_similarity": 0.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "IndexError)", + "got": "IndexError)", + "got_raw": "IndexError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "expected_result_identity", + "got": "expected_result_identity", + "got_raw": "expected_result_identity", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "argoverse/av2-api", + "expected": "city_SE3_ego_expected.rotation.matrix())", + "got": "city_SE3_ego.rotation)", + "got_raw": "city_SE3_ego.rotation)", + "exact_match": false, + "code_bleu": 0.25406637407730737, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "argoverse/av2-api", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "arogozhnikov/einops", + "expected": "x.dtype", + "got": "x.dtype", + "got_raw": "x.dtype", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "arogozhnikov/einops", + "expected": "(10 * 30 * 40, 10)", + "got": "(16, 8)", + "got_raw": "(16, 8)", + "exact_match": false, + "code_bleu": 0.18037106415283274, + "edit_similarity": 0.4 + }, + { + "repo": "arogozhnikov/einops", + "expected": "einops.EinopsError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.4666666666666667 + }, + { + "repo": "arogozhnikov/einops", + "expected": "EinopsError)", + "got": "EinopsError)", + "got_raw": "EinopsError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "arogozhnikov/einops", + "expected": "result[2, 3, 1]", + "got": "result[2, 3, 1]", + "got_raw": "result[2, 3, 1]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "arogozhnikov/einops", + "expected": "(10, 10, 30, 40)", + "got": "(10, 10, 30, 40)", + "got_raw": "(10, 10, 30, 40)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "arogozhnikov/einops", + "expected": "dict(a=1, b=20, c=30, d=40)", + "got": "parsed2", + "got_raw": "parsed2", + "exact_match": false, + "code_bleu": 0.027777777777777776, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "arogozhnikov/einops", + "expected": "ValueError, match=r\"^Einsum pattern\")", + "got": "ValueError, match=r\"^Einsum pattern must contain '->'\")", + "got_raw": "ValueError, match=r\"^Einsum pattern must contain '->'\")", + "exact_match": false, + "code_bleu": 0.7071067811865475, + "edit_similarity": 0.8043478260869565 + }, + { + "repo": "askui/python-sdk", + "expected": "second_id", + "got": "second_id", + "got_raw": "second_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "askui/python-sdk", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "askui/python-sdk", + "expected": "entry", + "got": "entry", + "got_raw": "entry", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "askui/python-sdk", + "expected": "OSError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.631578947368421 + }, + { + "repo": "askui/python-sdk", + "expected": "rendered", + "got": "rendered", + "got_raw": "rendered", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "askui/python-sdk", + "expected": "sorted(files)", + "got": "files", + "got_raw": "files", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "askui/python-sdk", + "expected": "scenario.tags", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.30120066786994826, + "edit_similarity": 0.0 + }, + { + "repo": "askui/python-sdk", + "expected": "\"PNG\"", + "got": "\"PNG\"", + "got_raw": "\"PNG\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "askui/python-sdk", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "askui/python-sdk", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "askui/python-sdk", + "expected": "dumped", + "got": "dumped", + "got_raw": "dumped", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "askui/python-sdk", + "expected": "mock_toolbox", + "got": "mock_toolbox", + "got_raw": "mock_toolbox", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "askui/python-sdk", + "expected": "\"OnlyName\"", + "got": "\"OnlyName\"", + "got_raw": "\"OnlyName\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "askui/python-sdk", + "expected": "second_id", + "got": "second_id", + "got_raw": "second_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "askui/python-sdk", + "expected": "params.steps", + "got": "[ScenarioStep(keyword=\"Given\", text=\"foo\")]", + "got_raw": "[ScenarioStep(keyword=\"Given\", text=\"foo\")]", + "exact_match": false, + "code_bleu": 0.08737167851715875, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "askui/python-sdk", + "expected": "img", + "got": "img", + "got_raw": "img", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "askui/python-sdk", + "expected": "ValueError)", + "got": "QueryNoResponseError)", + "got_raw": "QueryNoResponseError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "askui/python-sdk", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "askui/python-sdk", + "expected": "result_traj[0]", + "got": "result_traj[0]", + "got_raw": "result_traj[0]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "askui/python-sdk", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asottile/pyupgrade", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asottile/pyupgrade", + "expected": "s", + "got": "s", + "got_raw": "s", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asottile/pyupgrade", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asottile/pyupgrade", + "expected": "s", + "got": "s", + "got_raw": "s", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asottile/pyupgrade", + "expected": "s", + "got": "s", + "got_raw": "s", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asottile/pyupgrade", + "expected": "s", + "got": "s", + "got_raw": "s", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asottile/pyupgrade", + "expected": "s", + "got": "s", + "got_raw": "s", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asottile/pyupgrade", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asottile/pyupgrade", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asottile/pyupgrade", + "expected": "s", + "got": "s", + "got_raw": "s", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asottile/pyupgrade", + "expected": "s", + "got": "s", + "got_raw": "s", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asteroid-team/asteroid", + "expected": "est_targets.shape", + "got": "est_targets.shape", + "got_raw": "est_targets.shape", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asteroid-team/asteroid", + "expected": "RuntimeError)", + "got": "RuntimeError)", + "got_raw": "RuntimeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asteroid-team/asteroid", + "expected": "model(y))", + "got": "model(y))", + "got_raw": "model(y))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asteroid-team/asteroid", + "expected": "mixtures.shape", + "got": "mixtures.shape", + "got_raw": "mixtures.shape", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asteroid-team/asteroid", + "expected": "(batch, n_frames, 2 * n_units)", + "got": "(batch, n_frames, n_units)", + "got_raw": "(batch, n_frames, n_units)", + "exact_match": false, + "code_bleu": 0.5951601314210918, + "edit_similarity": 0.9285714285714286 + }, + { + "repo": "asteroid-team/asteroid", + "expected": "arbitrary_last_dim)", + "got": "arbitrary_last_dim)", + "got_raw": "arbitrary_last_dim)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asteroid-team/asteroid", + "expected": "min_idx_hun)", + "got": "min_idx_hun)", + "got_raw": "min_idx_hun)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asteroid-team/asteroid", + "expected": "type(asteroid_optim)", + "got": "type(asteroid_optim)", + "got_raw": "type(asteroid_optim)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asteroid-team/asteroid", + "expected": "w_src_wrapper(est_targets, targets, return_est=True)[1])", + "got": "w_src_wrapper(est_targets, targets, return_est=True)[1])", + "got_raw": "w_src_wrapper(est_targets, targets, return_est=True)[1])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "asteroid-team/asteroid", + "expected": "consistent_est_sources.sum(dim))", + "got": "consistent_est_sources)", + "got_raw": "consistent_est_sources)", + "exact_match": false, + "code_bleu": 0.16988673071467442, + "edit_similarity": 0.8363636363636363 + }, + { + "repo": "asteroid-team/asteroid", + "expected": "w_sum / n_src)", + "got": "w_sum)", + "got_raw": "w_sum)", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.6 + }, + { + "repo": "asteroid-team/asteroid", + "expected": "cnn.torch_complex_from_reim(inp.real + 1, inp.imag + 1))", + "got": "inp + 1)", + "got_raw": "inp + 1)", + "exact_match": false, + "code_bleu": 0.16637806616154058, + "edit_similarity": 0.25 + }, + { + "repo": "auth0/auth0-python", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "auth0/auth0-python", + "expected": "{ \"client_id\": \"cid\", \"client_secret\": \"csecret\", \"email\": \"a@b.com\", \"send\": \"snd\", \"connection\": \"email\", })", + "got": "{\"client_secret\": \"csecret\", \"email\": \"a@b.com\", \"send\": \"snd\"})", + "got_raw": "{\"client_secret\": \"csecret\", \"email\": \"a@b.com\", \"send\": \"snd\"})", + "exact_match": false, + "code_bleu": 0.5104379784655113, + "edit_similarity": 0.735632183908046 + }, + { + "repo": "auth0/auth0-python", + "expected": "[(\"hello_world[hello]\", \"world\")]", + "got": "[(\"hello_world\", {\"hello\": \"world\"})]", + "got_raw": "[(\"hello_world\", {\"hello\": \"world\"})]", + "exact_match": false, + "code_bleu": 0.20504572236241866, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "auth0/auth0-python", + "expected": "fnmatch(kwargs[\"data\"][\"client_assertion\"], \"*.*.*\"))", + "got": "kwargs[\"headers\"][\"Content-Type\"] == \"application/x-www-form-urlencoded\")", + "got_raw": "kwargs[\"headers\"][\"Content-Type\"] == \"application/x-www-form-urlencoded\")", + "exact_match": false, + "code_bleu": 0.17350022661435424, + "edit_similarity": 0.36507936507936506 + }, + { + "repo": "auth0/auth0-python", + "expected": "{\"client_id\": \"cid\", \"token\": \"tkn\"})", + "got": "{\"client_id\": \"cid\", \"token\": \"tkn\"})", + "got_raw": "{\"client_id\": \"cid\", \"token\": \"tkn\"})", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "auth0/auth0-python", + "expected": "{ \"client_id\": \"cid\", \"email\": \"a@b.com\", \"connection\": \"conn\", })", + "got": "{\"connection\": \"conn\", \"email\": \"a@b.com\", \"password\": \"pswd\"})", + "got_raw": "{\"connection\": \"conn\", \"email\": \"a@b.com\", \"password\": \"pswd\"})", + "exact_match": false, + "code_bleu": 0.505010117783677, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "auth0/auth0-python", + "expected": "{ \"client_id\": \"cid\", \"email\": \"a@b.com\", \"send\": \"snd\", \"connection\": \"email\", })", + "got": "{\"email\": \"a@b.com\", \"send\": \"snd\"})", + "got_raw": "{\"email\": \"a@b.com\", \"send\": \"snd\"})", + "exact_match": false, + "code_bleu": 0.39991351876399606, + "edit_similarity": 0.6101694915254238 + }, + { + "repo": "auth0/auth0-python", + "expected": "403)", + "got": "403)", + "got_raw": "403)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "auth0/auth0-python", + "expected": "args[0])", + "got": "args[0])", + "got_raw": "args[0])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "auth0/auth0-python", + "expected": "\"Request data does not match expected data after JSON serialization.\")", + "got": "expected_data)", + "got_raw": "expected_data)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.30952380952380953 + }, + { + "repo": "aymenfurter/microagents", + "expected": "closest_agent)", + "got": "closest_agent)", + "got_raw": "closest_agent)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aymenfurter/microagents", + "expected": "self.mock_openai_wrapper)", + "got": "self.mock_openai_wrapper)", + "got_raw": "self.mock_openai_wrapper)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aymenfurter/microagents", + "expected": "purposes)", + "got": "purposes)", + "got_raw": "purposes)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "aymenfurter/microagents", + "expected": "self.dummy_agent)", + "got": "self.dummy_agent)", + "got_raw": "self.dummy_agent)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bayespy/bayespy", + "expected": "Mu)", + "got": "0)", + "got_raw": "0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4 + }, + { + "repo": "bayespy/bayespy", + "expected": "())", + "got": "())", + "got_raw": "())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bayespy/bayespy", + "expected": "np.einsum('mn->', 0.5*np.ones(np.shape(tau))))", + "got": "m", + "got_raw": "m", + "exact_match": false, + "code_bleu": 0.023809523809523808, + "edit_similarity": 0.0425531914893617 + }, + { + "repo": "bayespy/bayespy", + "expected": "())", + "got": "(3,))", + "got_raw": "(3,))", + "exact_match": false, + "code_bleu": 0.3860973950960897, + "edit_similarity": 0.75 + }, + { + "repo": "bayespy/bayespy", + "expected": "(5,))", + "got": "(5,))", + "got_raw": "(5,))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bayespy/bayespy", + "expected": "1)", + "got": "3)", + "got_raw": "3)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "bayespy/bayespy", + "expected": "Lambda)", + "got": "X)", + "got_raw": "X)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "bayespy/bayespy", + "expected": "np.einsum('mn,ni->i', tau*y, a))", + "got": "ensordot(tau*y, a, axes=1))", + "got_raw": "ensordot(tau*y, a, axes=1))", + "exact_match": false, + "code_bleu": 0.4160751652217845, + "edit_similarity": 0.4745762711864407 + }, + { + "repo": "bayespy/bayespy", + "expected": "4)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "bayespy/bayespy", + "expected": "(9,))", + "got": "(9,))", + "got_raw": "(9,))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bayespy/bayespy", + "expected": "true_plates)", + "got": "true_plates)", + "got_raw": "true_plates)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bayespy/bayespy", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/beartype", + "expected": "TypeHint(None)", + "got": "TypeHint(str)", + "got_raw": "TypeHint(str)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7407407407407407 + }, + { + "repo": "beartype/beartype", + "expected": "Class", + "got": "Class", + "got_raw": "Class", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/beartype", + "expected": "NoneTypes", + "got": "NoneTypes", + "got_raw": "NoneTypes", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/beartype", + "expected": "beartype()", + "got": "beartype)", + "got_raw": "beartype)", + "exact_match": false, + "code_bleu": 0.39640237166757364, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "beartype/beartype", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/beartype", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/beartype", + "expected": "''", + "got": "f'{TEST_TARGET_STR}With his still soul.'", + "got_raw": "f'{TEST_TARGET_STR}With his still soul.'", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "beartype/beartype", + "expected": "'package_name'", + "got": "'package_name.subpackage_name.submodule_name'", + "got_raw": "'package_name.subpackage_name.submodule_name'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4745762711864407 + }, + { + "repo": "beartype/beartype", + "expected": "(str, int)", + "got": "hint=Dict[str, List[int]], args_len=2, exception_prefix='')", + "got_raw": "\n hint=Dict[str, List[int]], args_len=2, exception_prefix='')", + "exact_match": false, + "code_bleu": 0.09134423666564473, + "edit_similarity": 0.2608695652173913 + }, + { + "repo": "beartype/beartype", + "expected": "'Shook'", + "got": "who_had_power)", + "got_raw": "who_had_power)", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "beartype/beartype", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/beartype", + "expected": "84 + 1j", + "got": "1.0)", + "got_raw": "1.0)", + "exact_match": false, + "code_bleu": 0.2259005009024612, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "beartype/beartype", + "expected": "True", + "got": "now_float_above_thy_darkness)", + "got_raw": "now_float_above_thy_darkness)", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.06060606060606061 + }, + { + "repo": "beartype/beartype", + "expected": "beartype(isha)", + "got": "await beartype(isha)", + "got_raw": "await beartype(isha)", + "exact_match": false, + "code_bleu": 0.7598356856515925, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "beartype/beartype", + "expected": "INDENT_LEVEL_TO_CODE[2]", + "got": "INDENT_LEVEL_TO_CODE[2]", + "got_raw": "INDENT_LEVEL_TO_CODE[2]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/beartype", + "expected": "data_type", + "got": "data_type", + "got_raw": "data_type", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/beartype", + "expected": "is_stdout_terminal()", + "got": "is_stdout_terminal()", + "got_raw": "is_stdout_terminal()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/beartype", + "expected": "TypeIs", + "got": "TypeIs", + "got_raw": "TypeIs", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/beartype", + "expected": "''", + "got": "''", + "got_raw": "''", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/beartype", + "expected": "iter", + "got": "iter", + "got_raw": "iter", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/plum", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/plum", + "expected": "()", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "beartype/plum", + "expected": "\"Docs\"", + "got": "\"Docs\"", + "got_raw": "\"Docs\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/plum", + "expected": "Method(int, Signature(int), function_name=\"int\", return_type=int)", + "got": "m", + "got_raw": "m", + "exact_match": false, + "code_bleu": 0.029411764705882353, + "edit_similarity": 0.030303030303030304 + }, + { + "repo": "beartype/plum", + "expected": "v", + "got": "v", + "got_raw": "v", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/plum", + "expected": "C", + "got": "C[int, int, int]", + "got_raw": "C[int, int, int]", + "exact_match": false, + "code_bleu": 0.16036590969929357, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "beartype/plum", + "expected": "\"f(int)\"", + "got": "\"f(int)\"", + "got_raw": "\"f(int)\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/plum", + "expected": "Number(2)", + "got": "Number(2)", + "got_raw": "Number(2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/plum", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/plum", + "expected": "\"1\"", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/plum", + "expected": "\"/path/to/file\"", + "got": "\"/path/to/file\"", + "got_raw": "\"/path/to/file\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/plum", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/plum", + "expected": "t2", + "got": "t2", + "got_raw": "t2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/plum", + "expected": "\"f(1)\"", + "got": "\"f(1)\"", + "got_raw": "\"f(1)\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/plum", + "expected": "copy(m)", + "got": "copy(m)", + "got_raw": "copy(m)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/plum", + "expected": "(1, 1)", + "got": "(1, 1)", + "got_raw": "(1, 1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/plum", + "expected": "\"float or str\"", + "got": "\"float or str\"", + "got_raw": "\"float or str\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "beartype/plum", + "expected": "\"first\\n\\nsecond\"", + "got": "\"first\\n\\nsecond\\n\\nthird\"", + "got_raw": "\"first\\n\\nsecond\\n\\nthird\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7906976744186046 + }, + { + "repo": "beartype/plum", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "tables", + "got": "tables", + "got_raw": "tables", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "DocumentModel", + "got": "DocumentModel", + "got_raw": "DocumentModel", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "full_fields", + "got": "full_fields", + "got_raw": "full_fields", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"AdminAuditLog\"", + "got": "\"AuditLog\"", + "got_raw": "\"AuditLog\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"/login\"", + "got": "\"/admin/oauth2/token\"", + "got_raw": "\"/admin/oauth2/token\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3448275862068966 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"int\"", + "got": "\"int\"", + "got_raw": "\"int\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "\"redis\"", + "got": "\"redis\"", + "got_raw": "\"redis\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "expected_value", + "got": "expected_value", + "got_raw": "expected_value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "str(config.admin_engine.url)", + "got": "config.admin_db_url", + "got_raw": "config.admin_db_url", + "exact_match": false, + "code_bleu": 0.2229763340630102, + "edit_similarity": 0.6808510638297872 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "mock_event", + "got": "mock_event", + "got_raw": "mock_event", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/crudadmin", + "expected": "session_id)", + "got": "session_id)", + "got_raw": "session_id)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "ModelTest", + "got": "ModelTest", + "got_raw": "ModelTest", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "names", + "got": "names", + "got_raw": "names", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "sorted(tier_ids)", + "got": "sorted(tier_ids, reverse=True)", + "got_raw": "sorted(tier_ids, reverse=True)", + "exact_match": false, + "code_bleu": 0.3155984539112945, + "edit_similarity": 0.6956521739130435 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "len(test_data) - 1", + "got": "len([item for item in test_data if item[\"name\"] != test_data[0][\"name\"]])", + "got_raw": "len([item for item in test_data if item[\"name\"] != test_data[0][\"name\"]])", + "exact_match": false, + "code_bleu": 0.07575731225158963, + "edit_similarity": 0.32967032967032966 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "UserModel", + "got": "UserModel", + "got_raw": "UserModel", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "data", + "got": "data", + "got_raw": "data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "\"456\"", + "got": "\"456\"", + "got_raw": "\"456\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benavlabs/fastcrud", + "expected": "tester_data[\"uuid\"]", + "got": "tester_data[\"uuid\"]", + "got_raw": "tester_data[\"uuid\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benbusby/whoogle-search", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benbusby/whoogle-search", + "expected": "hrefs", + "got": "hrefs", + "got_raw": "hrefs", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benbusby/whoogle-search", + "expected": "valid_lang_keys", + "got": "valid_lang_keys", + "got_raw": "valid_lang_keys", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benbusby/whoogle-search", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "benbusby/whoogle-search", + "expected": "302", + "got": "200", + "got_raw": "200", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "benbusby/whoogle-search", + "expected": "''.join(divs)", + "got": "divs", + "got_raw": "divs", + "exact_match": false, + "code_bleu": 0.09910059291689341, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "bethgelab/foolbox", + "expected": "fbn.accuracy(fmodel, x, y)", + "got": "fbn.accuracy(fmodel, x, y)", + "got_raw": "fbn.accuracy(fmodel, x, y)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bethgelab/foolbox", + "expected": "(10, 5, 3)", + "got": "(10, 5, 3)", + "got_raw": "(10, 5, 3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bethgelab/foolbox", + "expected": "fbn.accuracy( fmodel, x, target_classes )", + "got": "fbn.accuracy(fmodel, x, target_classes)", + "got_raw": "fbn.accuracy(fmodel, x, target_classes)", + "exact_match": false, + "code_bleu": 1.0, + "edit_similarity": 0.975 + }, + { + "repo": "bethgelab/foolbox", + "expected": "ValueError, match=\"starting_points\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.46808510638297873 + }, + { + "repo": "bethgelab/foolbox", + "expected": "repr(distances[p])", + "got": "str(distances[p])", + "got_raw": "str(distances[p])", + "exact_match": false, + "code_bleu": 0.8408964152537145, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "bethgelab/foolbox", + "expected": "ValueError, match=\"channel_axis\")", + "got": "ValueError, match=\"data_format\")", + "got_raw": "ValueError, match=\"data_format\")", + "exact_match": false, + "code_bleu": 0.6434588841607617, + "edit_similarity": 0.7076923076923077 + }, + { + "repo": "bethgelab/foolbox", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bethgelab/foolbox", + "expected": "x.shape", + "got": "x.shape", + "got_raw": "x.shape", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "[\"metrics\"]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.39640237166757364, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "\"11\"", + "got": "\"14\"", + "got_raw": "\"14\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "[0, 1]", + "got": "list(range(2))", + "got_raw": "list(range(2))", + "exact_match": false, + "code_bleu": 0.1561969968460128, + "edit_similarity": 0.0 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "()", + "got": "()", + "got_raw": "()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "Exception)", + "got": "ImpossibleFilterError)", + "got_raw": "ImpossibleFilterError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.1875 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "[{\"a\": 1.0, \"b\": 2.0, \"rowid\": 0}]", + "got": "[(1, 2)]", + "got_raw": "[(1, 2)]", + "exact_match": false, + "code_bleu": 0.10308475862607012, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "InvalidPattern)", + "got": "InvalidPattern)", + "got_raw": "InvalidPattern)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "1", + "got": "start", + "got_raw": "start", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "{}", + "got": "{1: Operator.LIKE}", + "got_raw": "{1: Operator.LIKE}", + "exact_match": false, + "code_bleu": 0.20556680845025982, + "edit_similarity": 0.2 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "[(\"bob\", \"Italy\")]", + "got": "[(\"bob\", \"Italy\")]", + "got_raw": "[(\"bob\", \"Italy\")]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "[(version,)]", + "got": "[version]", + "got_raw": "[version]", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "{\"column1\": {(Operator.EQ, 3)}}", + "got": "{(\"column1\", \"=\", 3)}", + "got_raw": "{(\"column1\", \"=\", 3)}", + "exact_match": false, + "code_bleu": 0.3234032153502924, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "[{\"rowid\": 0}]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.16988673071467442, + "edit_similarity": 0.25 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "\"1.0\"", + "got": "\"1.0\"", + "got_raw": "\"1.0\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "[(\"Sam\", \"(949) 555-1234\", \"Irvine\", \"Solutions Architect\")]", + "got": "[(\"Sam\", \"(949) 555-1234\", \"Irvine\", \"Solutions Architect\")]", + "got_raw": "[(\"Sam\", \"(949) 555-1234\", \"Irvine\", \"Solutions Architect\")]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "betodealmeida/shillelagh", + "expected": "\"5\"", + "got": "\"13:14:05.001\"", + "got_raw": "\"13:14:05.001\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "bitcart/bitcart", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bitcart/bitcart", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bitcart/bitcart", + "expected": "\"test\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bitcart/bitcart", + "expected": "method)", + "got": "method[\"id\"])", + "got_raw": "method[\"id\"])", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.7 + }, + { + "repo": "bitcart/bitcart", + "expected": "\"BTC (\u26a1)\"", + "got": "\"BTC\"", + "got_raw": "\"BTC\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "bitcart/bitcart", + "expected": "\"Bitcart Merchants API\"", + "got": "\"Bitcart Merchants API\"", + "got_raw": "\"Bitcart Merchants API\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bitcart/bitcart", + "expected": "\"-\"", + "got": "\"-\"", + "got_raw": "\"-\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bitcart/bitcart", + "expected": "j", + "got": "j", + "got_raw": "j", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bitcart/bitcart", + "expected": "\"Hello {{var}}!\"", + "got": "\"Hello {{var}}!\"", + "got_raw": "\"Hello {{var}}!\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bitsandbytes-foundation/bitsandbytes", + "expected": "deserialized.state", + "got": "deserialized.state", + "got_raw": "deserialized.state", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bitsandbytes-foundation/bitsandbytes", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bitsandbytes-foundation/bitsandbytes", + "expected": "fx_ours)", + "got": "fx_ours)", + "got_raw": "fx_ours)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bitsandbytes-foundation/bitsandbytes", + "expected": "deq_c)", + "got": "deq_c)", + "got_raw": "deq_c)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bitsandbytes-foundation/bitsandbytes", + "expected": "dict_keys_after", + "got": "dict_keys_after", + "got_raw": "dict_keys_after", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bitsandbytes-foundation/bitsandbytes", + "expected": "RuntimeError)", + "got": "RuntimeError)", + "got_raw": "RuntimeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bitsandbytes-foundation/bitsandbytes", + "expected": "n_cases", + "got": "n_cases", + "got_raw": "n_cases", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bitsandbytes-foundation/bitsandbytes", + "expected": "ref_output.device", + "got": "ref_output.device", + "got_raw": "ref_output.device", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bitsandbytes-foundation/bitsandbytes", + "expected": "A1)", + "got": "A1)", + "got_raw": "A1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bitsandbytes-foundation/bitsandbytes", + "expected": "deep_copy.state", + "got": "deep_copy.state", + "got_raw": "deep_copy.state", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bitsandbytes-foundation/bitsandbytes", + "expected": "\"libbitsandbytes_rocm72\"", + "got": "\"rocm-7.2\"", + "got_raw": "\"rocm-7.2\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "{\"/page\"}", + "got": "[(\"page\", \"section\")]", + "got_raw": "[(\"page\", \"section\")]", + "exact_match": false, + "code_bleu": 0.1561969968460128, + "edit_similarity": 0.4 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "\"DONE\"", + "got": "\"DONE\"", + "got_raw": "\"DONE\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "edges", + "got": "edges", + "got_raw": "edges", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "dims", + "got": "dims", + "got_raw": "dims", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "1", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "\"dev\"", + "got": "\"dev\"", + "got_raw": "\"dev\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "result.columns", + "got": "result.columns", + "got_raw": "result.columns", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "\"line\"", + "got": "\"line\"", + "got_raw": "\"line\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "7", + "got": "7", + "got_raw": "7", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "\"value\"", + "got": "\"value\"", + "got_raw": "\"value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "boringdata/boring-semantic-layer", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bottlepy/bottle", + "expected": "test())", + "got": "test())", + "got_raw": "test())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bottlepy/bottle", + "expected": "lines)", + "got": "lines)", + "got_raw": "lines)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bottlepy/bottle", + "expected": "404)", + "got": "404)", + "got_raw": "404)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bottlepy/bottle", + "expected": "test())", + "got": "test())", + "got_raw": "test())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bottlepy/bottle", + "expected": "list(m.iterallitems()))", + "got": "list(m.items()))", + "got_raw": "list(m.items()))", + "exact_match": false, + "code_bleu": 0.6580370064762462, + "edit_similarity": 0.8205128205128205 + }, + { + "repo": "bottlepy/bottle", + "expected": "m.getall('a'))", + "got": "list(m.getlist('a')))", + "got_raw": "list(m.getlist('a')))", + "exact_match": false, + "code_bleu": 0.403527863746399, + "edit_similarity": 0.6857142857142857 + }, + { + "repo": "bottlepy/bottle", + "expected": "my_decorator in self.app.plugins)", + "got": "installed in self.app.plugins)", + "got_raw": "installed in self.app.plugins)", + "exact_match": false, + "code_bleu": 0.8633400213704505, + "edit_similarity": 0.6984126984126984 + }, + { + "repo": "bottlepy/bottle", + "expected": "ValueError)", + "got": "bottle.RouteBuildError, build, 'introute')", + "got_raw": "bottle.RouteBuildError, build, 'introute')", + "exact_match": false, + "code_bleu": 0.16036590969929357, + "edit_similarity": 0.3018867924528302 + }, + { + "repo": "bottlepy/bottle", + "expected": "'aaa)(?:_bbb)')", + "got": "'(?:aaa)(?:_bbb)')", + "got_raw": "'(?:aaa)(?:_bbb)')", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "bottlepy/bottle", + "expected": "KeyError)", + "got": "KeyError)", + "got_raw": "KeyError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bottlepy/bottle", + "expected": "byte_iter.closed)", + "got": "test_iter.closed)", + "got_raw": "test_iter.closed)", + "exact_match": false, + "code_bleu": 0.668740304976422, + "edit_similarity": 0.8823529411764706 + }, + { + "repo": "bottlepy/bottle", + "expected": "test2())", + "got": "test2())", + "got_raw": "test2())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bottlepy/bottle", + "expected": "\"text\")", + "got": "\"text\")", + "got_raw": "\"text\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bottlepy/bottle", + "expected": "plugin2 not in self.app.plugins)", + "got": "plugin2 in self.app.plugins)", + "got_raw": "plugin2 in self.app.plugins)", + "exact_match": false, + "code_bleu": 0.7903464044617434, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "bottlepy/bottle", + "expected": "t)", + "got": "t)", + "got_raw": "t)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bottlepy/bottle", + "expected": "test_file)", + "got": "bottle.tob(test_file))", + "got_raw": "bottle.tob(test_file))", + "exact_match": false, + "code_bleu": 0.24446151121745047, + "edit_similarity": 0.625 + }, + { + "repo": "bottlepy/bottle", + "expected": "good)", + "got": "good)", + "got_raw": "good)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bottlepy/bottle", + "expected": "True)", + "got": "True)", + "got_raw": "True)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bottlepy/bottle", + "expected": "'value3')", + "got": "'value3')", + "got_raw": "'value3')", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "buiapp/reaktiv", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "buiapp/reaktiv", + "expected": "15", + "got": "15", + "got_raw": "15", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "buiapp/reaktiv", + "expected": "3.0", + "got": "3.0", + "got_raw": "3.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "buiapp/reaktiv", + "expected": "cleanup_ran", + "got": "cleanup_ran", + "got_raw": "cleanup_ran", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "buiapp/reaktiv", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "buiapp/reaktiv", + "expected": "10.1", + "got": "10.1", + "got_raw": "10.1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "buiapp/reaktiv", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "buiapp/reaktiv", + "expected": "40", + "got": "40", + "got_raw": "40", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "buiapp/reaktiv", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "buiapp/reaktiv", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "buiapp/reaktiv", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "buiapp/reaktiv", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bytedance/trae-agent", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bytedance/trae-agent", + "expected": "\"new content\")", + "got": "str(self.test_file), \"new content\")", + "got_raw": "str(self.test_file), \"new content\")", + "exact_match": false, + "code_bleu": 0.18575057999133596, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "bytedance/trae-agent", + "expected": "written_data[\"config\"])", + "got": "written_data)", + "got_raw": "written_data)", + "exact_match": false, + "code_bleu": 0.23784142300054423, + "edit_similarity": 0.7222222222222222 + }, + { + "repo": "bytedance/trae-agent", + "expected": "\"\")", + "got": "\"Temperature: 25\u00b0C, Humidity: 60%\")", + "got_raw": "\"Temperature: 25\u00b0C, Humidity: 60%\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.15789473684210525 + }, + { + "repo": "bytedance/trae-agent", + "expected": "\"Bob\")", + "got": "\"Bob\")", + "got_raw": "\"Bob\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bytedance/trae-agent", + "expected": "\"A test tool\")", + "got": "self.mock_tool.description)", + "got_raw": "self.mock_tool.description)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.2926829268292683 + }, + { + "repo": "bytedance/trae-agent", + "expected": "restart_result.output.lower())", + "got": "restart_result)", + "got_raw": "restart_result)", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "bytedance/trae-agent", + "expected": "\"Charlie\")", + "got": "\"Charlie\")", + "got_raw": "\"Charlie\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bytedance/trae-agent", + "expected": "self.agent.cli_console)", + "got": "self.agent.llm_client.client)", + "got_raw": "self.agent.llm_client.client)", + "exact_match": false, + "code_bleu": 0.46713797772820004, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "bytedance/trae-agent", + "expected": "\"test_provider\")", + "got": "\"test_provider\")", + "got_raw": "\"test_provider\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "bytedance/trae-agent", + "expected": "\"true\")", + "got": "True)", + "got_raw": "True)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "caronc/apprise-api", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "caronc/apprise-api", + "expected": "405", + "got": "405", + "got_raw": "405", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "caronc/apprise-api", + "expected": "{ \"persistent_storage\": False, \"can_write_config\": True, \"can_write_attach\": True, \"details\": [\"OK\"], }", + "got": "{ \"persistent_storage\": False, \"can_write_config\": True, \"can_write_attach\": True, \"details\": [ \"STORE_PERMISSION_ISSUE\", ], }", + "got_raw": "{ \"persistent_storage\": False, \"can_write_config\": True, \"can_write_attach\": True, \"details\": [ \"STORE_PERMISSION_ISSUE\", ], }", + "exact_match": false, + "code_bleu": 0.8232490471721702, + "edit_similarity": 0.8908296943231441 + }, + { + "repo": "caronc/apprise-api", + "expected": "payload", + "got": "payload", + "got_raw": "payload", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "caronc/apprise-api", + "expected": "403", + "got": "403", + "got_raw": "403", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "caronc/apprise-api", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "caronc/apprise-api", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "caronc/apprise-api", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "caronc/apprise-api", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cased/kit", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cased/kit", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cased/kit", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cased/kit", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cased/kit", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cased/kit", + "expected": "{}", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "cased/kit", + "expected": "graph", + "got": "graph", + "got_raw": "graph", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cased/kit", + "expected": "canon2", + "got": "canon2", + "got_raw": "canon2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cased/kit", + "expected": "result.output", + "got": "result.stdout", + "got_raw": "result.stdout", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "cased/kit", + "expected": "prompt", + "got": "prompt", + "got_raw": "prompt", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cased/kit", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cased/kit", + "expected": "data", + "got": "data", + "got_raw": "data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cased/kit", + "expected": "\"*.py\"", + "got": "\"*.py\"", + "got_raw": "\"*.py\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cased/kit", + "expected": "url", + "got": "url", + "got_raw": "url", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cased/kit", + "expected": "graph", + "got": "graph", + "got_raw": "graph", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cased/kit", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cased/kit", + "expected": "id4", + "got": "id4", + "got_raw": "id4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cased/kit", + "expected": "7", + "got": "7", + "got_raw": "7", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cased/kit", + "expected": "names", + "got": "names", + "got_raw": "names", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cased/kit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chiphuyen/sniffly", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chiphuyen/sniffly", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chiphuyen/sniffly", + "expected": "total_tokens)", + "got": "self.statistics['overview'])", + "got_raw": "self.statistics['overview'])", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.24390243902439024 + }, + { + "repo": "chiphuyen/sniffly", + "expected": "\"Should have input tokens\")", + "got": "\"Total input tokens should be greater than 0\")", + "got_raw": "\"Total input tokens should be greater than 0\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4931506849315068 + }, + { + "repo": "chiphuyen/sniffly", + "expected": "\"Error count should be identical\")", + "got": "(baseline_errors)", + "got_raw": "(baseline_errors)", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "chiphuyen/sniffly", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chiphuyen/sniffly", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chiphuyen/sniffly", + "expected": "\"Should have 109 user messages\")", + "got": "(message_types.get('user', 0), 10)", + "got_raw": "(message_types.get('user', 0), 10)", + "exact_match": false, + "code_bleu": 0.10390302174233558, + "edit_similarity": 0.2727272727272727 + }, + { + "repo": "chiphuyen/sniffly", + "expected": "\"Message count should not change with optimizations\")", + "got": "(len(messages), len(self.baseline_messages))", + "got_raw": "(len(messages), len(self.baseline_messages))", + "exact_match": false, + "code_bleu": 0.09552040806823771, + "edit_similarity": 0.30927835051546393 + }, + { + "repo": "chiphuyen/sniffly", + "expected": "interruption)", + "got": "interruption)", + "got_raw": "interruption)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chiphuyen/sniffly", + "expected": "\"Should have data for 2025-06-10\")", + "got": "\"Should have data for 2025-06-10\")", + "got_raw": "\"Should have data for 2025-06-10\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chiphuyen/sniffly", + "expected": "\"Should process same number of projects\")", + "got": "(len(seq_results), len(par_results))", + "got_raw": "(len(seq_results), len(par_results))", + "exact_match": false, + "code_bleu": 0.11390778025531027, + "edit_similarity": 0.2077922077922078 + }, + { + "repo": "chiphuyen/sniffly", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chiphuyen/sniffly", + "expected": "\"Initialization should be under 100ms\")", + "got": "\"Initialization should complete within 0.1 seconds\")", + "got_raw": "\"Initialization should complete within 0.1 seconds\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6593406593406593 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "2", + "got": "13", + "got_raw": "13", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "2", + "got": "13", + "got_raw": "13", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "2", + "got": "13", + "got_raw": "13", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "original", + "got": "original", + "got_raw": "original", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "\"v1\"", + "got": "\"v1\"", + "got_raw": "\"v1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "\"my-key\"", + "got": "\"my-key\"", + "got_raw": "\"my-key\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "1", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "chunk_size + 20", + "got": "chunk_size", + "got_raw": "chunk_size", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.8 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "model", + "got": "model", + "got_raw": "model", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "\"r\")", + "got": "\"r\")", + "got_raw": "\"r\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "chunk.text", + "got": "chunk.text", + "got_raw": "chunk.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "\"el\"", + "got": "\"el\"", + "got_raw": "\"el\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chonkie-inc/chonkie", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "christabor/flask_jsondash", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "christabor/flask_jsondash", + "expected": "27017", + "got": "27017", + "got_raw": "27017", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "christabor/flask_jsondash", + "expected": "app.config.get('JSONDASH_GLOBAL_USER')", + "got": "'admin'", + "got_raw": "'admin'", + "exact_match": false, + "code_bleu": 0.0625, + "edit_similarity": 0.08888888888888889 + }, + { + "repo": "christabor/flask_jsondash", + "expected": "100", + "got": "100", + "got_raw": "100", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "christabor/flask_jsondash", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "christabor/flask_jsondash", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "christabor/flask_jsondash", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "christabor/flask_jsondash", + "expected": "'Username'", + "got": "'admin'", + "got_raw": "'admin'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "christabor/flask_jsondash", + "expected": "result.output", + "got": "result.output", + "got_raw": "result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "christabor/flask_jsondash", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "christabor/flask_jsondash", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chuyangliu/snake", + "expected": "Pos(-10, 10)", + "got": "Pos(-10, 10)", + "got_raw": "Pos(-10, 10)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chuyangliu/snake", + "expected": "m_copy.point(Pos(i, j)).type", + "got": "m_copy.point(Pos(i, j)).type", + "got_raw": "m_copy.point(Pos(i, j)).type", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chuyangliu/snake", + "expected": "Direc.UP", + "got": "Direc.LEFT", + "got_raw": "Direc.LEFT", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "chuyangliu/snake", + "expected": "PointType.WALL", + "got": "PointType.WALL", + "got_raw": "PointType.WALL", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chuyangliu/snake", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "chuyangliu/snake", + "expected": "Direc.RIGHT", + "got": "Direc.RIGHT", + "got_raw": "Direc.RIGHT", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "coderamp-labs/gitingest", + "expected": "InvalidGitHubTokenError)", + "got": "InvalidGitHubTokenError)", + "got_raw": "InvalidGitHubTokenError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "coderamp-labs/gitingest", + "expected": "content", + "got": "content", + "got_raw": "content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "coderamp-labs/gitingest", + "expected": "token)", + "got": "token)", + "got_raw": "token)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "coderamp-labs/gitingest", + "expected": "content_with_ignore", + "got": "content_with_ignore", + "got_raw": "content_with_ignore", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "coderamp-labs/gitingest", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "coderamp-labs/gitingest", + "expected": "structure", + "got": "structure", + "got_raw": "structure", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "coderamp-labs/gitingest", + "expected": "stderr_lines", + "got": "stderr_lines", + "got_raw": "stderr_lines", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cohere-ai/cohere-python", + "expected": "dict)", + "got": "dict)", + "got_raw": "dict)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cohere-ai/cohere-python", + "expected": "256)", + "got": "256)", + "got_raw": "256)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cohere-ai/cohere-python", + "expected": "str(request.url))", + "got": "captured_aws_request_kwargs[\"headers\"][\"host\"])", + "got_raw": "captured_aws_request_kwargs[\"headers\"][\"host\"])", + "exact_match": false, + "code_bleu": 0.16036590969929357, + "edit_similarity": 0.34375 + }, + { + "repo": "cohere-ai/cohere-python", + "expected": "\"text-generation\" in events)", + "got": "events)", + "got_raw": "events)", + "exact_match": false, + "code_bleu": 0.3535533905932738, + "edit_similarity": 0.4 + }, + { + "repo": "cohere-ai/cohere-python", + "expected": "response)", + "got": "response)", + "got_raw": "response)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "cohere-ai/cohere-python", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "commaai/laika", + "expected": "sat_info_orbit[2])", + "got": "sat_info_orbit[2])", + "got_raw": "sat_info_orbit[2])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "commaai/laika", + "expected": "prn)", + "got": "prn)", + "got_raw": "prn)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "commaai/laika", + "expected": "os.path.exists(file.filepath_attempt))", + "got": "os.path.exists(file.filepath_attempt))", + "got_raw": "os.path.exists(file.filepath_attempt))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "commaai/laika", + "expected": "14)", + "got": "0)", + "got_raw": "0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4 + }, + { + "repo": "commaai/laika", + "expected": "1.7378390525714509)", + "got": "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "got_raw": "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.027210884353741496 + }, + { + "repo": "commaai/opendbc", + "expected": "expected[checksum_field]", + "got": "expected[checksum_field]", + "got_raw": "expected[checksum_field]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "commaai/opendbc", + "expected": "std", + "got": "std", + "got_raw": "std", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "commaai/opendbc", + "expected": "should_match", + "got": "should_match", + "got_raw": "should_match", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "commaai/opendbc", + "expected": "aeb_event == 0)", + "got": "[])", + "got_raw": "[])", + "exact_match": false, + "code_bleu": 0.2710806010829534, + "edit_similarity": 0.1111111111111111 + }, + { + "repo": "commaai/opendbc", + "expected": "self.MIN_GAS)", + "got": "self.MIN_GAS)", + "got_raw": "self.MIN_GAS)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "commaai/opendbc", + "expected": "self._tx(self._acc_cancel_msg(False)))", + "got": "self._tx(self._acc_cancel_msg(False)))", + "got_raw": "self._tx(self._acc_cancel_msg(False)))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "commaai/opendbc", + "expected": "len(result)", + "got": "len(result)", + "got_raw": "len(result)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "commaai/opendbc", + "expected": "self._rx(self._speed_msg(0)))", + "got": "self.safety.rx_hook(self._speed_msg(10)))", + "got_raw": "self.safety.rx_hook(self._speed_msg(10)))", + "exact_match": false, + "code_bleu": 0.4480304273880272, + "edit_similarity": 0.8 + }, + { + "repo": "commaai/opendbc", + "expected": "self._tx(self._button_msg(resume=True)))", + "got": "self._tx(self._button_msg(resume=True)))", + "got_raw": "self._tx(self._button_msg(resume=True)))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "commaai/opendbc", + "expected": "self._tx(tester_present))", + "got": "self.safety.test(tester_present))", + "got_raw": "self.safety.test(tester_present))", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.7931034482758621 + }, + { + "repo": "commaai/opendbc", + "expected": "i % 256", + "got": "i + 1)", + "got_raw": "i + 1)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "commaai/opendbc", + "expected": "{(b\"CV1-CV000\", None)}", + "got": "{(b\"CV1\", b\"99110\")}", + "got_raw": "{(b\"CV1\", b\"99110\")}", + "exact_match": false, + "code_bleu": 0.32466791547509893, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "commaai/opendbc", + "expected": "self.safety.get_controls_allowed())", + "got": "self.safety.controls_allowed())", + "got_raw": "self.safety.controls_allowed())", + "exact_match": false, + "code_bleu": 0.5969491792019646, + "edit_similarity": 0.9393939393939394 + }, + { + "repo": "commaai/opendbc", + "expected": "car_params.centerToFront", + "got": "car_params.centerToFront", + "got_raw": "car_params.centerToFront", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "commaai/opendbc", + "expected": "len(codes)", + "got": "len(codes)", + "got_raw": "len(codes)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "commaai/opendbc", + "expected": "should_fail", + "got": "should_fail", + "got_raw": "should_fail", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "commaai/opendbc", + "expected": "{platform}", + "got": "car_fw", + "got_raw": "car_fw", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.25 + }, + { + "repo": "commaai/opendbc", + "expected": "yr2)", + "got": "yr2)", + "got_raw": "yr2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "commaai/opendbc", + "expected": "self._rx(msg))", + "got": "self._rx(msg))", + "got_raw": "self._rx(msg))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "commaai/opendbc", + "expected": "1", + "got": "len(expected)", + "got_raw": "len(expected)", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.0 + }, + { + "repo": "containers/ramalama", + "expected": "16", + "got": "16", + "got_raw": "16", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "containers/ramalama", + "expected": "\"All done\"", + "got": "\"All done\"", + "got_raw": "\"All done\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "containers/ramalama", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "containers/ramalama", + "expected": "22", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "containers/ramalama", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "containers/ramalama", + "expected": "sio.read()", + "got": "sio.read()", + "got_raw": "sio.read()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "containers/ramalama", + "expected": "content", + "got": "content", + "got_raw": "content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "containers/ramalama", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "containers/ramalama", + "expected": "True", + "got": "expected", + "got_raw": "expected", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "containers/ramalama", + "expected": "provider", + "got": "provider", + "got_raw": "provider", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "containers/ramalama", + "expected": "8", + "got": "5", + "got_raw": "5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "containers/ramalama", + "expected": "value", + "got": "value", + "got_raw": "value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "containers/ramalama", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "containers/ramalama", + "expected": "42", + "got": "42", + "got_raw": "42", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "containers/ramalama", + "expected": "content", + "got": "content", + "got_raw": "content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "containers/ramalama", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "containers/ramalama", + "expected": "info", + "got": "info", + "got_raw": "info", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "containers/ramalama", + "expected": "\"gpt-4o-mini\"", + "got": "\"podman\"", + "got_raw": "\"podman\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "containers/ramalama", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "containers/ramalama", + "expected": "\"1024\"", + "got": "1024", + "got_raw": "1024", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8 + }, + { + "repo": "copier-org/copier", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "copier-org/copier", + "expected": "defaults", + "got": "defaults", + "got_raw": "defaults", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "copier-org/copier", + "expected": "\"P2\"", + "got": "\"P2\"", + "got_raw": "\"P2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "copier-org/copier", + "expected": "\"v1\"", + "got": "sha", + "got_raw": "sha", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "copier-org/copier", + "expected": "\"v1\"", + "got": "\"v1\"", + "got_raw": "\"v1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "copier-org/copier", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "copier-org/copier", + "expected": "\"la vie en rose\"", + "got": "\"la vie en rose\"", + "got_raw": "\"la vie en rose\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "copier-org/copier", + "expected": "copier.errors.TaskError)", + "got": "copier.CopyError)", + "got_raw": "copier.CopyError)", + "exact_match": false, + "code_bleu": 0.3387551654364098, + "edit_similarity": 0.6829268292682927 + }, + { + "repo": "copier-org/copier", + "expected": "str(tpl1)", + "got": "str(tpl1)", + "got_raw": "str(tpl1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "copier-org/copier", + "expected": "values[:1]", + "got": "values", + "got_raw": "values", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.75 + }, + { + "repo": "copier-org/copier", + "expected": "\"v1\"", + "got": "\"v2\"", + "got_raw": "\"v2\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "copier-org/copier", + "expected": "\"SKIPPED\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "copier-org/copier", + "expected": "{\"owner2\": owner2}", + "got": "{\"owner2\": owner2}", + "got_raw": "{\"owner2\": owner2}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "copier-org/copier", + "expected": "\"bar\"", + "got": "\"bar\"", + "got_raw": "\"bar\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "copier-org/copier", + "expected": "\"v2.0.0.alpha1\"", + "got": "\"v2.0.0.dev1\"", + "got_raw": "\"v2.0.0.dev1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "copier-org/copier", + "expected": "\"2\"", + "got": "\"2\"", + "got_raw": "\"2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "copier-org/copier", + "expected": "\"0.2.0\"", + "got": "\"0.2.0\"", + "got_raw": "\"0.2.0\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "copier-org/copier", + "expected": "normalized", + "got": "normalized", + "got_raw": "normalized", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "copier-org/copier", + "expected": "\"task 1\\ntask 2\\n\"", + "got": "\"created\"", + "got_raw": "\"created\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2962962962962963 + }, + { + "repo": "copier-org/copier", + "expected": "control", + "got": "control", + "got_raw": "control", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "2", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "[9, 8, 7]", + "got": "[9, 8, 7]", + "got_raw": "[9, 8, 7]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "\"a\"", + "got": "\"a\"", + "got_raw": "\"a\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "\"test\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "{\"A\", \"B\"}", + "got": "{\"A\", \"B\"}", + "got_raw": "{\"A\", \"B\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "\"low\"", + "got": "\"low\"", + "got_raw": "\"low\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "s2", + "got": "s2", + "got_raw": "s2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "Token(\"PLUS\", \"+\")", + "got": "Token(\"PLUS\", \"+\")", + "got_raw": "Token(\"PLUS\", \"+\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "\"B\"", + "got": "'c'", + "got_raw": "'c'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "42", + "got": "42", + "got_raw": "42", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "\"C\"", + "got": "\"C\"", + "got_raw": "\"C\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crazyguitar/pysheeet", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "headers", + "got": "headers", + "got_raw": "headers", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "status", + "got": "status", + "got_raw": "status", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "urllib.error.URLError)", + "got": "urllib.error.URLError)", + "got_raw": "urllib.error.URLError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "'8'", + "got": "8", + "got_raw": "8", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "\"New York\"", + "got": "\"New York\"", + "got_raw": "\"New York\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "\"param1\"", + "got": "\"param1\"", + "got_raw": "\"param1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "ET.ParseError)", + "got": "urllib.error.URLError)", + "got_raw": "urllib.error.URLError)", + "exact_match": false, + "code_bleu": 0.24446151121745047, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "OSError)", + "got": "OSError)", + "got_raw": "OSError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "\"\"", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "website)", + "got": "website)", + "got_raw": "website)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "\"en\"", + "got": "\"en\"", + "got_raw": "\"en\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "crewAIInc/crewAI-tools", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "amount", + "got": "amount", + "got_raw": "amount", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "bob", + "got": "bob", + "got_raw": "bob", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "value", + "got": "value", + "got_raw": "value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "alice", + "got": "alice", + "got_raw": "alice", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "tuple(rewards)", + "got": "tuple(rewards)", + "got_raw": "tuple(rewards)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "charlie", + "got": "charlie", + "got_raw": "charlie", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "10 ** 20", + "got": "1000000", + "got_raw": "1000000", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.4 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "10 ** 19", + "got": "10 ** 19", + "got_raw": "10 ** 19", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "time", + "got": "time", + "got_raw": "time", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "REWARD", + "got": "REWARD", + "got_raw": "REWARD", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "accounts[3]", + "got": "accounts[3]", + "got_raw": "accounts[3]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "REWARD", + "got": "REWARD", + "got_raw": "REWARD", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "balance", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "bob", + "got": "bob", + "got_raw": "bob", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "coin_a", + "got": "coin_a", + "got_raw": "coin_a", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "balance", + "got": "balance", + "got_raw": "balance", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "curvefi/curve-dao-contracts", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "d0c-s4vage/lookatme", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "d0c-s4vage/lookatme", + "expected": "title", + "got": "title", + "got_raw": "title", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "d0c-s4vage/lookatme", + "expected": "b\"link\"", + "got": "b\"link\"", + "got_raw": "b\"link\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "d0c-s4vage/lookatme", + "expected": "AssertionError)", + "got": "marshmallow.exceptions.ValidationError)", + "got_raw": "marshmallow.exceptions.ValidationError)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.4074074074074074 + }, + { + "repo": "d0c-s4vage/lookatme", + "expected": "res.output", + "got": "", + "got_raw": "", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "danielgtaylor/python-betterproto", + "expected": "new_again", + "got": "new_again", + "got_raw": "new_again", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "danielgtaylor/python-betterproto", + "expected": "RecursiveMessage()", + "got": "msg", + "got_raw": "msg", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "danielgtaylor/python-betterproto", + "expected": "\"List[str]\"", + "got": "\"List[str]\"", + "got_raw": "\"List[str]\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "danielgtaylor/python-betterproto", + "expected": "data", + "got": "data", + "got_raw": "data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "danielgtaylor/python-betterproto", + "expected": "\"FOUR\"", + "got": "\"FOUR\"", + "got_raw": "\"FOUR\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "danielgtaylor/python-betterproto", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "danielgtaylor/python-betterproto", + "expected": "value", + "got": "value", + "got_raw": "value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "danielgtaylor/python-betterproto", + "expected": "multi_byte", + "got": "multi_byte", + "got_raw": "multi_byte", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "danielgtaylor/python-betterproto", + "expected": "expected_responses", + "got": "expected_responses", + "got_raw": "expected_responses", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "danielgtaylor/python-betterproto", + "expected": "ref_obj", + "got": "ref_obj", + "got_raw": "ref_obj", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "danielgtaylor/python-betterproto", + "expected": "[name]", + "got": "[name]", + "got_raw": "[name]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "danielgtaylor/python-betterproto", + "expected": "\"ZERO\"", + "got": "\"choice\" in Test(choice=Choice.ONE).to_dict()", + "got_raw": "\"choice\" in Test(choice=Choice.ONE).to_dict()", + "exact_match": false, + "code_bleu": 0.07432998184513635, + "edit_similarity": 0.0784313725490196 + }, + { + "repo": "danielgtaylor/python-betterproto", + "expected": "deepcopied", + "got": "deepcopied", + "got_raw": "deepcopied", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "danielgtaylor/python-betterproto", + "expected": "deadline._timestamp)", + "got": "0.0, abs=1e-3)", + "got_raw": "0.0, abs=1e-3)", + "exact_match": false, + "code_bleu": 0.16036590969929357, + "edit_similarity": 0.17647058823529413 + }, + { + "repo": "danielgtaylor/python-betterproto", + "expected": "'\"_p__.Message\"'", + "got": "\"_p__.Message\"", + "got_raw": "\"_p__.Message\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "danielgtaylor/python-betterproto", + "expected": "[\"ZERO\"]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.39640237166757364, + "edit_similarity": 0.4 + }, + { + "repo": "danielgtaylor/python-betterproto", + "expected": "repr_value", + "got": "repr_value", + "got_raw": "repr_value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "1 if contains else 0", + "got": "NLargest)", + "got_raw": "NLargest)", + "exact_match": false, + "code_bleu": 0.18072040072196896, + "edit_similarity": 0.20689655172413793 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "sql_string", + "got": "sql_string", + "got_raw": "sql_string", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "expect_filters", + "got": "expect_filters", + "got_raw": "expect_filters", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "df[[\"id\"]])", + "got": "df[[\"id\"]])", + "got_raw": "df[[\"id\"]])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "c.schema", + "got": "c.schemas", + "got_raw": "c.schemas", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.9411764705882353 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "c.sql(\"SELECT * FROM df\"))", + "got": "c.sql(\"SELECT * FROM foo.df\"))", + "got_raw": "c.sql(\"SELECT * FROM foo.df\"))", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.9285714285714286 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "\"\"\" SELECT AVG(c) FROM df2 \"\"\")", + "got": "\"\"\"SELECT", + "got_raw": "\"\"\"SELECT\n d, SUM(c), SUM(user_id)\n FROM df2\n WHERE d IS NOT NULL -- dask behaves differently on NaNs in groupbys\n GROUP BY d\n ORDER BY SUM(c)\n LIMIT 10\"\"\")", + "exact_match": false, + "code_bleu": 0.3388507513536918, + "edit_similarity": 0.45 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "get_system_sample(ddf, 0.99999, 10))", + "got": "get_system_sample(ddf, 0.00001, 10),", + "got_raw": "get_system_sample(ddf, 0.00001, 10),", + "exact_match": false, + "code_bleu": 0.5253819788848316, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "\"\"\" SELECT CASE WHEN user_id = 3 THEN 4 ELSE user_id END FROM df2 \"\"\")", + "got": "\"\"\"SELECT * FROM df1 WHERE user_id = 1\"\"\")", + "got_raw": "\"\"\"SELECT * FROM df1 WHERE user_id = 1\"\"\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.5178571428571429 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "AttributeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "\"string\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "c.DEFAULT_SCHEMA_NAME", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.0 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "\"\"\" SELECT * FROM df1 \"\"\")", + "got": "\"\"\"SELECT * FROM foo\"\"\")", + "got_raw": "\"\"\"SELECT * FROM foo\"\"\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.88 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "RuntimeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "False", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "second_mock", + "got": "second_mock", + "got_raw": "second_mock", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "\"\"\" SELECT df2.user_id, df2.d FROM df2 \"\"\")", + "got": "\"\"\"SELECT * FROM df1 WHERE df1.a > 10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "got_raw": "\"\"\"SELECT * FROM df1 WHERE df1.a > 10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "exact_match": false, + "code_bleu": 0.10390302174233558, + "edit_similarity": 0.1958762886597938 + }, + { + "repo": "dask-contrib/dask-sql", + "expected": "result)", + "got": "result)", + "got_raw": "result)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "\"test-key\"", + "got": "\"test-key\"", + "got_raw": "\"test-key\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "{\"start\": \"A\", \"C\": \"C\"}", + "got": "{\"start\": \"B\", \"C\": \"C\"}", + "got_raw": "{\"start\": \"B\", \"C\": \"C\"}", + "exact_match": false, + "code_bleu": 0.6580370064762462, + "edit_similarity": 0.9583333333333334 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "\"test\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "\"COSINE\"", + "got": "\"Cosine\"", + "got_raw": "\"Cosine\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.375 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "\"alpha\"", + "got": "\"alpha\"", + "got_raw": "\"alpha\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "\"Hello, world!\"", + "got": "\"Hello, world!\"", + "got_raw": "\"Hello, world!\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "Memory()", + "got": "Memory()", + "got_raw": "Memory()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "18", + "got": "9", + "got_raw": "9", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "\"model\"", + "got": "\"system\"", + "got_raw": "\"system\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "\"test_value\"", + "got": "\"test_value\"", + "got_raw": "\"test_value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "\"test-key\"", + "got": "\"test-key\"", + "got_raw": "\"test-key\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "\"Hello, world!\"", + "got": "\"Hello, world!\"", + "got_raw": "\"Hello, world!\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "\"user\"", + "got": "\"user\"", + "got_raw": "\"user\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "\"test_tool\"", + "got": "\"test_tool\"", + "got_raw": "\"test_tool\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "[TextBlock(content=\"Mock response\")]", + "got": "\"Mock response\"", + "got_raw": "\"Mock response\"", + "exact_match": false, + "code_bleu": 0.07432544468767006, + "edit_similarity": 0.5882352941176471 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "\"test_elements\"", + "got": "\"test_collection\"", + "got_raw": "\"test_collection\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.625 + }, + { + "repo": "datapizza-labs/datapizza-ai", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "api.SyftrStudyStatus.STOPPED", + "got": "api.SyftrStudyStatus.STOPPED", + "got_raw": "api.SyftrStudyStatus.STOPPED", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "df.columns", + "got": "df.columns", + "got_raw": "df.columns", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "pareto_df.columns", + "got": "pareto_df.columns", + "got_raw": "pareto_df.columns", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datarobot/syftr", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "datmo/datmo", + "expected": "ref_id_2", + "got": "ref_id_2", + "got_raw": "ref_id_2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datmo/datmo", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datmo/datmo", + "expected": "v", + "got": "v", + "got_raw": "v", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datmo/datmo", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datmo/datmo", + "expected": "tags", + "got": "tags", + "got_raw": "tags", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datmo/datmo", + "expected": "self.base_controller.home", + "got": "self.temp_dir", + "got_raw": "self.temp_dir", + "exact_match": false, + "code_bleu": 0.3567621345008163, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "datmo/datmo", + "expected": "set(ids)", + "got": "set(ids)", + "got_raw": "set(ids)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datmo/datmo", + "expected": "model2.id", + "got": "model2.id", + "got_raw": "model2.id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datmo/datmo", + "expected": "self.user_input_dict['name']", + "got": "self.user_input_dict[\"name\"]", + "got_raw": "self.user_input_dict[\"name\"]", + "exact_match": false, + "code_bleu": 0.6434588841607617, + "edit_similarity": 0.9285714285714286 + }, + { + "repo": "datmo/datmo", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datmo/datmo", + "expected": "f2", + "got": "f2", + "got_raw": "f2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datmo/datmo", + "expected": "default_name", + "got": "default_name", + "got_raw": "default_name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datmo/datmo", + "expected": "result.duration", + "got": "result.duration", + "got_raw": "result.duration", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datmo/datmo", + "expected": "code_obj", + "got": "code_obj", + "got_raw": "code_obj", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datmo/datmo", + "expected": "set(ids)", + "got": "set(ids)", + "got_raw": "set(ids)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datmo/datmo", + "expected": "v", + "got": "v", + "got_raw": "v", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datmo/datmo", + "expected": "test_ports", + "got": "test_ports", + "got_raw": "test_ports", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datmo/datmo", + "expected": "\"a\"", + "got": "\"a\"", + "got_raw": "\"a\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datmo/datmo", + "expected": "set(ids)", + "got": "set(ids)", + "got_raw": "set(ids)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "datmo/datmo", + "expected": "3", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "david-lev/pywa", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "david-lev/pywa", + "expected": "\"value\"", + "got": "\"value\"", + "got_raw": "\"value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "david-lev/pywa", + "expected": "ValueError, match=\"Expected 2 cards in Carousel parameters, but got 1.\")", + "got": "pytest.raises(ValueError, match=\"Missing parameters for: CarouselCard$\")", + "got_raw": "pytest.raises(ValueError, match=\"Missing parameters for: CarouselCard$\")", + "exact_match": false, + "code_bleu": 0.3672056269893592, + "edit_similarity": 0.5277777777777778 + }, + { + "repo": "david-lev/pywa", + "expected": "{\"type\": \"string\"}", + "got": "\"string\"", + "got_raw": "\"string\"", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "david-lev/pywa", + "expected": "class_with_parentheses_args", + "got": "class_with_parentheses_args", + "got_raw": "class_with_parentheses_args", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "david-lev/pywa", + "expected": "data.get(\"href\")", + "got": "data.get(\"href\")", + "got_raw": "data.get(\"href\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "david-lev/pywa", + "expected": "h", + "got": "h", + "got_raw": "h", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "david-lev/pywa", + "expected": "\"abc\"", + "got": "\"abc\"", + "got_raw": "\"abc\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "david-lev/pywa", + "expected": "\"manual\"", + "got": "\"manual\"", + "got_raw": "\"manual\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "david-lev/pywa", + "expected": "TypeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "\"dbos-toolbox\"", + "got": "\"dbos-toolbox\"", + "got_raw": "\"dbos-toolbox\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "wfid", + "got": "wfid", + "got_raw": "wfid", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "\"1\"", + "got": "\"test_transaction\"", + "got_raw": "\"test_transaction\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "\"None!\"", + "got": "str(\"Trouble?\")", + "got_raw": "str(\"Trouble?\")", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "\"value\"", + "got": "\"value\"", + "got_raw": "\"value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "27", + "got": "config[\"sys_db_pool_size\"]", + "got_raw": "config[\"sys_db_pool_size\"]", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.0 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "2)", + "got": "3)", + "got_raw": "3)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "4", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "\"Yay!\"", + "got": "\"Yay!\"", + "got_raw": "\"Yay!\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "\"plain_password\")", + "got": "config[\"database_url\"])", + "got_raw": "config[\"database_url\"])", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.3 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "\"a\"", + "got": "\"a\"", + "got_raw": "\"a\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "\"This is a secret_password test\")", + "got": "\"This is a secret_password test\")", + "got_raw": "\"This is a secret_password test\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "input_val", + "got": "input_val", + "got_raw": "input_val", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dbos-inc/dbos-transact-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "decoderesearch/circuit-tracer", + "expected": "2", + "got": "clt.n_layers", + "got_raw": "clt.n_layers", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "decoderesearch/circuit-tracer", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "decoderesearch/circuit-tracer", + "expected": "n_active", + "got": "n_active", + "got_raw": "n_active", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "decoderesearch/circuit-tracer", + "expected": "components", + "got": "components", + "got_raw": "components", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "decoderesearch/circuit-tracer", + "expected": "expected_token_strs[1]", + "got": "expected_token_strs[1]", + "got_raw": "expected_token_strs[1]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "decoderesearch/circuit-tracer", + "expected": "(batch_size * seq_len, 512)", + "got": "(test_input.shape[0], d_model)", + "got_raw": "(test_input.shape[0], d_model)", + "exact_match": false, + "code_bleu": 0.14991106946711685, + "edit_similarity": 0.3157894736842105 + }, + { + "repo": "decoderesearch/circuit-tracer", + "expected": "(len(feat_ids), lazy_clt.n_layers, lazy_clt.d_model)", + "got": "(4, lazy_clt.d_model)", + "got_raw": "(4, lazy_clt.d_model)", + "exact_match": false, + "code_bleu": 0.35355339059327373, + "edit_similarity": 0.547945205479452 + }, + { + "repo": "decoderesearch/circuit-tracer", + "expected": "str_without_acts", + "got": "str_without_acts", + "got_raw": "str_without_acts", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "decoderesearch/circuit-tracer", + "expected": "2", + "got": "clt.n_layers", + "got_raw": "clt.n_layers", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "decoderesearch/circuit-tracer", + "expected": "expected_state[\"W_dec\"].shape", + "got": "expected_state[\"W_dec\"].shape", + "got_raw": "expected_state[\"W_dec\"].shape", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "decoderesearch/circuit-tracer", + "expected": "(3,)", + "got": "(3,)", + "got_raw": "(3,)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedalus-labs/dedalus-sdk-python", + "expected": "\"a.b=c\"", + "got": "\"a.b=c\"", + "got_raw": "\"a.b=c\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedalus-labs/dedalus-sdk-python", + "expected": "obj2[\"foo\"][\"bar\"][0])", + "got": "obj2[\"foo\"][\"bar\"][0])", + "got_raw": "obj2[\"foo\"][\"bar\"][0])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedalus-labs/dedalus-sdk-python", + "expected": "\"org/project\"", + "got": "\"org/project\"", + "got_raw": "\"org/project\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedalus-labs/dedalus-sdk-python", + "expected": "\"github\"", + "got": "\"github\"", + "got_raw": "\"github\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedalus-labs/dedalus-sdk-python", + "expected": "15000", + "got": "15000", + "got_raw": "15000", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedalus-labs/dedalus-sdk-python", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedalus-labs/dedalus-sdk-python", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedalus-labs/dedalus-sdk-python", + "expected": "{\"foo\": \"bar\"}", + "got": "{\"foo\": \"bar\"}", + "got_raw": "{\"foo\": \"bar\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedalus-labs/dedalus-sdk-python", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedalus-labs/dedalus-sdk-python", + "expected": "\"a\"", + "got": "\"a\"", + "got_raw": "\"a\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedalus-labs/dedalus-sdk-python", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedalus-labs/dedalus-sdk-python", + "expected": "\"a=true\"", + "got": "\"a=1\"", + "got_raw": "\"a=1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "dedalus-labs/dedalus-sdk-python", + "expected": "\"hello!\"", + "got": "\"hello!\"", + "got_raw": "\"hello!\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedalus-labs/dedalus-sdk-python", + "expected": "\"v1.2.0\"", + "got": "\"v1.2.0\"", + "got_raw": "\"v1.2.0\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedalus-labs/dedalus-sdk-python", + "expected": "\"in=foo,bar\"", + "got": "\"in=foo%2Cbar\"", + "got_raw": "\"in=foo%2Cbar\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8461538461538461 + }, + { + "repo": "dedalus-labs/dedalus-sdk-python", + "expected": "_ENVELOPE_VERSION", + "got": "0x01", + "got_raw": "0x01", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "dedalus-labs/dedalus-sdk-python", + "expected": "\"ping\"", + "got": "\"ping\"", + "got_raw": "\"ping\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedalus-labs/dedalus-sdk-python", + "expected": "6", + "got": "6", + "got_raw": "6", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedupeio/dedupe", + "expected": "[0, 2]", + "got": "[0]", + "got_raw": "[0]", + "exact_match": false, + "code_bleu": 0.38336586254776345, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "dedupeio/dedupe", + "expected": "{\"0\"}", + "got": "set()", + "got_raw": "set()", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "dedupeio/dedupe", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedupeio/dedupe", + "expected": "{\"go\"}", + "got": "{\"go\"}", + "got_raw": "{\"go\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedupeio/dedupe", + "expected": "[\"deduplicat\", \"eduplicate\"]", + "got": "[\"deduplicat\"]", + "got_raw": "[\"deduplicat\"]", + "exact_match": false, + "code_bleu": 0.38336586254776345, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "dedupeio/dedupe", + "expected": "IndexError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6363636363636364 + }, + { + "repo": "dedupeio/dedupe", + "expected": "self.desired_scored_pairs[\"pairs\"])", + "got": "self.desired_scored_pairs[\"pairs\"])", + "got_raw": "self.desired_scored_pairs[\"pairs\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dedupeio/dedupe", + "expected": "{\"cc\"}", + "got": "{\"11cp\"}", + "got_raw": "{\"11cp\"}", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "dedupeio/dedupe", + "expected": "{frozenset([120, 125]), frozenset([130, 135])}", + "got": "{frozenset({\"100\", \"110\", \"120\", \"130\", \"140\"})}", + "got_raw": "{frozenset({\"100\", \"110\", \"120\", \"130\", \"140\"})}", + "exact_match": false, + "code_bleu": 0.15867278925431863, + "edit_similarity": 0.5106382978723404 + }, + { + "repo": "deepseek-ai/smallpond", + "expected": "[{\"a\": 1, \"b\": 4}, {\"a\": 2, \"b\": 5}]", + "got": "pd.DataFrame({\"a\": [2, 3], \"b\": [5, 6]})", + "got_raw": "pd.DataFrame({\"a\": [2, 3], \"b\": [5, 6]})", + "exact_match": false, + "code_bleu": 0.18556675930109115, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "deepseek-ai/smallpond", + "expected": "exec_plan.successful)", + "got": "exec_plan)", + "got_raw": "exec_plan)", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.6451612903225806 + }, + { + "repo": "deepseek-ai/smallpond", + "expected": "len(exec_plan.final_output.load_partitioned_datasets(npartitions, data_partition_column)))", + "got": "exec_plan)", + "got_raw": "exec_plan)", + "exact_match": false, + "code_bleu": 0.08494336535733721, + "edit_similarity": 0.2 + }, + { + "repo": "deepseek-ai/smallpond", + "expected": "len(partitioned_datasets))", + "got": "len(partitioned_datasets))", + "got_raw": "len(partitioned_datasets))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "deepseek-ai/smallpond", + "expected": "arrow_batch_output.num_rows)", + "got": "arrow_batch_output.num_rows)", + "got_raw": "arrow_batch_output.num_rows)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "deepseek-ai/smallpond", + "expected": "pa.table({\"c\": [5, 7, 9]})", + "got": "pa.table({\"c\": [5, 7, 9]})", + "got_raw": "pa.table({\"c\": [5, 7, 9]})", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "deepseek-ai/smallpond", + "expected": "sum(dataset.num_rows for dataset in partitioned_datasets))", + "got": "orig_dataset)", + "got_raw": "orig_dataset)", + "exact_match": false, + "code_bleu": 0.097688175620322, + "edit_similarity": 0.3380281690140845 + }, + { + "repo": "deepseek-ai/smallpond", + "expected": "get_nth_partition(items, 0, 2))", + "got": "get_nth_partition(items, 0, 2))", + "got_raw": "get_nth_partition(items, 0, 2))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "deepseek-ai/smallpond", + "expected": "sorted(final_table.column_names))", + "got": "len(final_table))", + "got_raw": "len(final_table))", + "exact_match": false, + "code_bleu": 0.3227150012892303, + "edit_similarity": 0.6 + }, + { + "repo": "deepseek-ai/smallpond", + "expected": "file.schema_arrow.metadata)", + "got": "file.schema_arrow.metadata)", + "got_raw": "file.schema_arrow.metadata)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "denizsafak/abogen", + "expected": "out", + "got": "out", + "got_raw": "out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "denizsafak/abogen", + "expected": "\"Fiction\"", + "got": "\"Fiction\"", + "got_raw": "\"Fiction\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "denizsafak/abogen", + "expected": "\"Summary text\"", + "got": "\"Summary text\"", + "got_raw": "\"Summary text\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "denizsafak/abogen", + "expected": "\"Alexandre Dumas\"", + "got": "[\"Alexandre Dumas\"]", + "got_raw": "[\"Alexandre Dumas\"]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.9444444444444444 + }, + { + "repo": "denizsafak/abogen", + "expected": "\"All Caps Title\"", + "got": "\"all caps title\"", + "got_raw": "\"all caps title\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8125 + }, + { + "repo": "denizsafak/abogen", + "expected": "parser.content_texts[id_1])", + "got": "parser.get_content(id_1))", + "got_raw": "parser.get_content(id_1))", + "exact_match": false, + "code_bleu": 0.2777619034011791, + "edit_similarity": 0.7307692307692307 + }, + { + "repo": "denizsafak/abogen", + "expected": "speakers", + "got": "speakers", + "got_raw": "speakers", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "denizsafak/abogen", + "expected": "normalize(\"2009\", cfg)", + "got": "normalize(\"2009\", cfg)", + "got_raw": "normalize(\"2009\", cfg)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "denizsafak/abogen", + "expected": "[\"1\"]", + "got": "[\"1\"]", + "got_raw": "[\"1\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "denizsafak/abogen", + "expected": "expected_root", + "got": "expected_root", + "got_raw": "expected_root", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "denizsafak/abogen", + "expected": "\"speaker_voice\"", + "got": "\"speaker_voice\"", + "got_raw": "\"speaker_voice\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "denizsafak/abogen", + "expected": "formula", + "got": "formula", + "got_raw": "formula", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "denizsafak/abogen", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "denizsafak/abogen", + "expected": "ids)", + "got": "ids)", + "got_raw": "ids)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "denizsafak/abogen", + "expected": "captured", + "got": "captured", + "got_raw": "captured", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "denizsafak/abogen", + "expected": "\"Test Author\")", + "got": "\"Test Author\")", + "got_raw": "\"Test Author\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "denizsafak/abogen", + "expected": "text1)", + "got": "text1)", + "got_raw": "text1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "denizsafak/abogen", + "expected": "\"Volescu\"", + "got": "\"Blue Leader\"", + "got_raw": "\"Blue Leader\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "denizsafak/abogen", + "expected": "PdfParser)", + "got": "PdfParser)", + "got_raw": "PdfParser)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "denizsafak/abogen", + "expected": "\"male\"", + "got": "\"male\"", + "got_raw": "\"male\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "descriptinc/audiotools", + "expected": "spk_batch", + "got": "spk_batch.convolve(ir)", + "got_raw": "spk_batch.convolve(ir)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.5806451612903226 + }, + { + "repo": "descriptinc/audiotools", + "expected": "sample_rate", + "got": "sample_rate", + "got_raw": "sample_rate", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "descriptinc/audiotools", + "expected": "1e-4", + "got": "0.01", + "got_raw": "0.01", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.25 + }, + { + "repo": "descriptinc/audiotools", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "descriptinc/audiotools", + "expected": "44100", + "got": "44100", + "got_raw": "44100", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "descriptinc/audiotools", + "expected": "signal_from_ffmpeg.signal_length", + "got": "signal_from_ffmpeg.signal_length", + "got_raw": "signal_from_ffmpeg.signal_length", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "descriptinc/audiotools", + "expected": "sig3", + "got": "sig3", + "got_raw": "sig3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dgasmith/opt_einsum", + "expected": "(2, 2)", + "got": "(2, 2)", + "got_raw": "(2, 2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dgasmith/opt_einsum", + "expected": "\"GEMM\"", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "dgasmith/opt_einsum", + "expected": "res_got3)", + "got": "res_got3)", + "got_raw": "res_got3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dgasmith/opt_einsum", + "expected": "expr.__str__()", + "got": "str(expr)", + "got_raw": "str(expr)", + "exact_match": false, + "code_bleu": 0.3418296051169873, + "edit_similarity": 0.43478260869565216 + }, + { + "repo": "dgasmith/opt_einsum", + "expected": "object", + "got": "np.float64", + "got_raw": "np.float64", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.25 + }, + { + "repo": "dgasmith/opt_einsum", + "expected": "TypeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "dgasmith/opt_einsum", + "expected": "0.0", + "got": "sparse_opt", + "got_raw": "sparse_opt", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "dgilland/pydash", + "expected": "_.initial(value)", + "got": "[1, 2, 3, 4]", + "got_raw": "[1, 2, 3, 4]", + "exact_match": false, + "code_bleu": 0.11868405219520975, + "edit_similarity": 0.0 + }, + { + "repo": "dgilland/pydash", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dgilland/pydash", + "expected": "_._", + "got": "_", + "got_raw": "_", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5 + }, + { + "repo": "dgilland/pydash", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dgilland/pydash", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "[{\"data\": \"test\"}]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.16988673071467442, + "edit_similarity": 0.2 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "actual", + "got": "actual", + "got_raw": "actual", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "extra_context", + "got": "extra_context", + "got_raw": "extra_context", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "(\"foo\",)", + "got": "(\"foo\",)", + "got_raw": "(\"foo\",)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "\"World\"", + "got": "\"Example\"", + "got_raw": "\"Example\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.375 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "\"X-UNICORN\"", + "got": "\"HTTP_X_UNICORN\"", + "got_raw": "\"HTTP_X_UNICORN\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7407407407407407 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "call_functions", + "got": "call_functions", + "got_raw": "call_functions", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "flavor.name", + "got": "flavor.name", + "got_raw": "flavor.name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "{\"NAME\": \"morphdom\"}", + "got": "settings.MORPHER", + "got_raw": "settings.MORPHER", + "exact_match": false, + "code_bleu": 0.22795070569547776, + "edit_similarity": 0.1111111111111111 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "component.integer", + "got": "set_property_value(component, \"int\", 99)", + "got_raw": "set_property_value(component, \"int\", 99)", + "exact_match": false, + "code_bleu": 0.16036590969929357, + "edit_similarity": 0.42105263157894735 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "grandparent", + "got": "grandparent", + "got_raw": "grandparent", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "flavor.pk", + "got": "flavor.pk", + "got_raw": "flavor.pk", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "parent", + "got": "parent", + "got_raw": "parent", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-commons/django-unicorn", + "expected": "html", + "got": "html", + "got_raw": "html", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-crispy-forms/django-crispy-forms", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-crispy-forms/django-crispy-forms", + "expected": "Exception)", + "got": "UnicodeEncodeError)", + "got_raw": "UnicodeEncodeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.13793103448275862 + }, + { + "repo": "django-crispy-forms/django-crispy-forms", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-crispy-forms/django-crispy-forms", + "expected": "7", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "django-crispy-forms/django-crispy-forms", + "expected": "\"legend\"", + "got": "\"legend\"", + "got_raw": "\"legend\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "CommandError, \"Unknown database non-existing_database\")", + "got": "CommandError as e:", + "got_raw": " CommandError as e:\n call_command('reset_schema', 'myapp', '--database', 'does_not_exist')", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.4657534246575342 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "active)", + "got": "active)", + "got_raw": "active)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "fake_user2[\"username\"]", + "got": "\"niccolas\"", + "got_raw": "\"niccolas\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.25 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "CommandError, match=\"Only available in debug mode\")", + "got": "CommandError)", + "got_raw": "CommandError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.40625 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "CommandError, \"Unknown database unknown\")", + "got": "CommandError, \"Unknown database type\")", + "got_raw": "CommandError, \"Unknown database type\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8607594936708861 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "m_stdout.getvalue())", + "got": "os.path.exists(os.path.join(JOBS_DIR, time_period)))", + "got_raw": "os.path.exists(os.path.join(JOBS_DIR, time_period)))", + "exact_match": false, + "code_bleu": 0.101824256461955, + "edit_similarity": 0.1388888888888889 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "\"-2\")", + "got": "\"\")", + "got_raw": "\"\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.75 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "str(\"Test Value\"))", + "got": "\"Test Value\")", + "got_raw": "\"Test Value\")", + "exact_match": false, + "code_bleu": 0.28284271247461906, + "edit_similarity": 0.8780487804878049 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "CommandError, \"1 errors found\")", + "got": "CommandError, \"Found errors in templates\")", + "got_raw": "CommandError, \"Found errors in templates\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7123287671232876 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "first or set())", + "got": "first)", + "got_raw": "first)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "\"TeslaCar\")", + "got": "\"Person\")", + "got_raw": "\"Person\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "CommandError, \"Default site with pk=321 does not exist\")", + "got": "CommandError, \"Site object with id 321 does not exist\")", + "got_raw": "CommandError, \"Site object with id 321 does not exist\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7927927927927928 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "m_stderr.getvalue())", + "got": "os.path.isfile(filepath))", + "got_raw": "os.path.isfile(filepath))", + "exact_match": false, + "code_bleu": 0.21105340631872635, + "edit_similarity": 0.35555555555555557 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "pdb.set_trace.called)", + "got": "pdb.set_trace.called)", + "got_raw": "pdb.set_trace.called)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "set())", + "got": "set())", + "got_raw": "set())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "output)", + "got": "output)", + "got_raw": "output)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "m_stdout.getvalue())", + "got": "m_stdout.getvalue())", + "got_raw": "m_stdout.getvalue())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "django-extensions/django-extensions", + "expected": "tuple)", + "got": "tuple)", + "got_raw": "tuple)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "doganarif/fastapi-radar", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "doganarif/fastapi-radar", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "doganarif/fastapi-radar", + "expected": "\"2\"", + "got": "\"2\"", + "got_raw": "\"2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "doganarif/fastapi-radar", + "expected": "\"abc123\"", + "got": "\"abc123\"", + "got_raw": "\"abc123\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "doganarif/fastapi-radar", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "doganarif/fastapi-radar", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "doganarif/fastapi-radar", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "doganarif/fastapi-radar", + "expected": "\"failed\"", + "got": "\"failed\"", + "got_raw": "\"failed\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "doganarif/fastapi-radar", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "dooit-org/dooit", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dooit-org/dooit", + "expected": "\"(icon) this is a [italic", + "got": "\"(icon) this is a test description 123\"", + "got_raw": "\"(icon) this is a test description 123\"", + "exact_match": false, + "code_bleu": 0.05555555555555555, + "edit_similarity": 0.625 + }, + { + "repo": "dooit-org/dooit", + "expected": "new_options[::-1]", + "got": "new_options", + "got_raw": "new_options", + "exact_match": false, + "code_bleu": 0.08494336535733721, + "edit_similarity": 0.7857142857142857 + }, + { + "repo": "dooit-org/dooit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dooit-org/dooit", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dooit-org/dooit", + "expected": "4", + "got": "6", + "got_raw": "6", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "dooit-org/dooit", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dooit-org/dooit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dreadnode/dyana", + "expected": "{\"os\": 1}", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.0 + }, + { + "repo": "dreadnode/dyana", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dreadnode/dyana", + "expected": "result[\"object_type\"]", + "got": "result", + "got_raw": "result", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "dreadnode/dyana", + "expected": "\"test-loader\"", + "got": "loader.name", + "got_raw": "loader.name", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.5 + }, + { + "repo": "dreadnode/dyana", + "expected": "False", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "dreadnode/dyana", + "expected": "\"1.0TiB\"", + "got": "'1.00 TiB'", + "got_raw": "'1.00 TiB'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "dreadnode/dyana", + "expected": "{\"timeout\": \"timeout reached\"}", + "got": "[\"timeout\"]", + "got_raw": "[\"timeout\"]", + "exact_match": false, + "code_bleu": 0.2710806010829534, + "edit_similarity": 0.43902439024390244 + }, + { + "repo": "dreadnode/dyana", + "expected": "\"my_model\"", + "got": "\"my_model\"", + "got_raw": "\"my_model\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dreadnode/dyana", + "expected": "\"/cache\"", + "got": "\"/cache\"", + "got_raw": "\"/cache\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dreadnode/dyana", + "expected": "\"rw\"", + "got": "\"rw\"", + "got_raw": "\"rw\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "_UnevaluatedForwardRefError)", + "got": "_UnevaluatedForwardRefError)", + "got_raw": "_UnevaluatedForwardRefError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"GlobalOtherClassBefore\"", + "got": "\"GlobalOtherClassBefore\"", + "got_raw": "\"GlobalOtherClassBefore\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"str\"", + "got": "\"str\"", + "got_raw": "\"str\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"members\"", + "got": "\"members\"", + "got_raw": "\"members\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "GlobalOtherModelAfter", + "got": "GlobalOtherModelAfter", + "got_raw": "GlobalOtherModelAfter", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"str\"", + "got": "\"str\"", + "got_raw": "\"str\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "predicate_fn", + "got": "predicate_fn", + "got_raw": "predicate_fn", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"Party\"", + "got": "\"ImportedRef\"", + "got_raw": "\"ImportedRef\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"GlobalWithFwdRefs\"", + "got": "\"GlobalWithFwdRefs\"", + "got_raw": "\"GlobalWithFwdRefs\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "\"cf-many\"", + "got": "\"cf-many\"", + "got_raw": "\"cf-many\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "msgspec_examples.Party", + "got": "Party", + "got_raw": "Party", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.37037037037037035 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "expected_dataclasses_models", + "got": "expected_dataclasses_models", + "got_raw": "expected_dataclasses_models", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "pydantic_examples.Party", + "got": "pydantic_examples.Party", + "got_raw": "pydantic_examples.Party", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "drivendataorg/erdantic", + "expected": "attrs_examples.Party", + "got": "Party", + "got_raw": "Party", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.4 + }, + { + "repo": "drvinceknight/Nashpy", + "expected": "(128, 2187)", + "got": "(2 ** 7, 3 ** 7)", + "got_raw": "(2 ** 7, 3 ** 7)", + "exact_match": false, + "code_bleu": 0.1354599427337814, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "drvinceknight/Nashpy", + "expected": "(1, number_of_rows + 1)", + "got": "(number_of_rows, number_of_rows)", + "got_raw": "(number_of_rows, number_of_rows)", + "exact_match": false, + "code_bleu": 0.2916059217599022, + "edit_similarity": 0.6545454545454545 + }, + { + "repo": "drvinceknight/Nashpy", + "expected": "GeneratorType)", + "got": "GeneratorType)", + "got_raw": "GeneratorType)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "drvinceknight/Nashpy", + "expected": "[(), (0,), (1,), (0, 1)])", + "got": "[[0], [1], [0, 1]])", + "got_raw": "[[0], [1], [0, 1]])", + "exact_match": false, + "code_bleu": 0.15094693751396754, + "edit_similarity": 0.5 + }, + { + "repo": "drvinceknight/Nashpy", + "expected": "len(population)", + "got": "len(population)", + "got_raw": "len(population)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "drvinceknight/Nashpy", + "expected": "(number_of_strategies + dimension, dimension + 1))", + "got": "number_of_strategies)", + "got_raw": "number_of_strategies)", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.5915492957746479 + }, + { + "repo": "drvinceknight/Nashpy", + "expected": "DeprecationWarning)", + "got": "DeprecationWarning)", + "got_raw": "DeprecationWarning)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "drvinceknight/Nashpy", + "expected": "{len(initial_population)}", + "got": "{len(last_generation)}", + "got_raw": "{len(last_generation)}", + "exact_match": false, + "code_bleu": 0.488923022434901, + "edit_similarity": 0.5957446808510638 + }, + { + "repo": "drvinceknight/Nashpy", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "drvinceknight/Nashpy", + "expected": "np.array_equal(halfspace[number_of_strategies:, :-1], -np.eye(dimension)))", + "got": "halfspace[0, 0]", + "got_raw": "halfspace[0, 0]", + "exact_match": false, + "code_bleu": 0.08616950394818627, + "edit_similarity": 0.29213483146067415 + }, + { + "repo": "drvinceknight/Nashpy", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "dylanljones/pyrekordbox", + "expected": "len(blues)", + "got": "len(blues)", + "got_raw": "len(blues)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dylanljones/pyrekordbox", + "expected": "times)", + "got": "times)", + "got_raw": "times)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dylanljones/pyrekordbox", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dylanljones/pyrekordbox", + "expected": "p", + "got": "p", + "got_raw": "p", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "RuntimeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"7.2\"", + "got": "\"6.10\"", + "got_raw": "\"6.10\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"testing\"", + "got": "\"testing\"", + "got_raw": "\"testing\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "value", + "got": "value", + "got_raw": "value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "val", + "got": "val", + "got_raw": "val", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "{\"name\": \"Bruno\", \"age\": 17})", + "got": "\"John\")", + "got_raw": "\"John\")", + "exact_match": false, + "code_bleu": 0.1074569931823542, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"testing\"", + "got": "\"testing\"", + "got_raw": "\"testing\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "42", + "got": "42", + "got_raw": "42", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "RuntimeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"Task2-2\"", + "got": "\"Task2-2\"", + "got_raw": "\"Task2-2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "settings.PORT", + "got": "settings.PORT", + "got_raw": "settings.PORT", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"blaz\"", + "got": "\"blaz\"", + "got_raw": "\"blaz\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "6", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "dynaconf/dynaconf", + "expected": "\"b\")", + "got": "\"b\")", + "got_raw": "\"b\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "TypeError)", + "got": "Base58ChecksumError)", + "got_raw": "Base58ChecksumError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4666666666666667 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "XlmAddr is XlmAddrEncoder)", + "got": "XlmAddr.IsOldAddrClass())", + "got_raw": "XlmAddr.IsOldAddrClass())", + "exact_match": false, + "code_bleu": 0.24446151121745047, + "edit_similarity": 0.5490196078431373 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "TypeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "TypeError)", + "got": "MoneroKeyError)", + "got_raw": "MoneroKeyError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.56 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "TypeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "monero_obj.PrimaryAddress())", + "got": "monero_obj.PrimaryAddress())", + "got_raw": "monero_obj.PrimaryAddress())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "str(mnemonic))", + "got": "mnemonic.ToStr())", + "got_raw": "mnemonic.ToStr())", + "exact_match": false, + "code_bleu": 0.3303164318013807, + "edit_similarity": 0.6451612903225806 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "path_elem.IsHard())", + "got": "path_elem.IsHard())", + "got_raw": "path_elem.IsHard())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "Bip32Depth(1) < Bip32Depth(2))", + "got": "Bip32KeyDataConst.ZERO == Bip32KeyDataConst.ZERO)", + "got_raw": "Bip32KeyDataConst.ZERO == Bip32KeyDataConst.ZERO)", + "exact_match": false, + "code_bleu": 0.127025927377108, + "edit_similarity": 0.43037974683544306 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "bip32_obj.Index())", + "got": "bip32_obj.Index())", + "got_raw": "bip32_obj.Index())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "int(elem))", + "got": "elem)", + "got_raw": "elem)", + "exact_match": false, + "code_bleu": 0.28284271247461906, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "binascii.hexlify(seed))", + "got": "seed)", + "got_raw": "seed)", + "exact_match": false, + "code_bleu": 0.20203050891044216, + "edit_similarity": 0.35714285714285715 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "ValueError)", + "got": "binascii.Error)", + "got_raw": "binascii.Error)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.5384615384615384 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "str(mnemonic))", + "got": "mnemonic.ToStr())", + "got_raw": "mnemonic.ToStr())", + "exact_match": false, + "code_bleu": 0.3303164318013807, + "edit_similarity": 0.6451612903225806 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "conv_addr)", + "got": "conv_addr)", + "got_raw": "conv_addr)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "curve_type)", + "got": "curve_type)", + "got_raw": "curve_type)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "P2SHAddr is P2SHAddrEncoder)", + "got": "P2SHAddr is P2SHAddrEncoder)", + "got_raw": "P2SHAddr is P2SHAddrEncoder)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "str(path_elem))", + "got": "path_elem.ToStr())", + "got_raw": "path_elem.ToStr())", + "exact_match": false, + "code_bleu": 0.3303164318013807, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "ebellocchia/bip_utils", + "expected": "test_bit_len // 8)", + "got": "test_bit_len // 8)", + "got_raw": "test_bit_len // 8)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ekzhu/datasketch", + "expected": "key in keys)", + "got": "key in keys)", + "got_raw": "key in keys)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ekzhu/datasketch", + "expected": "d[lm2])", + "got": "True)", + "got_raw": "True)", + "exact_match": false, + "code_bleu": 0.2149139863647084, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "ekzhu/datasketch", + "expected": "b1 < b2)", + "got": "b1 == b2)", + "got_raw": "b1 == b2)", + "exact_match": false, + "code_bleu": 0.3860973950960897, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "ekzhu/datasketch", + "expected": "m2)", + "got": "m2)", + "got_raw": "m2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ekzhu/datasketch", + "expected": "lm1.jaccard(lm2) < 1.0)", + "got": "lm1.jaccard(lm2) == 0.5)", + "got_raw": "lm1.jaccard(lm2) == 0.5)", + "exact_match": false, + "code_bleu": 0.5900468726392808, + "edit_similarity": 0.851063829787234 + }, + { + "repo": "ekzhu/datasketch", + "expected": "b1 < b2)", + "got": "b1 == b2)", + "got_raw": "b1 == b2)", + "exact_match": false, + "code_bleu": 0.3860973950960897, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "ekzhu/datasketch", + "expected": "lm2)", + "got": "lm2)", + "got_raw": "lm2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ekzhu/datasketch", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ekzhu/datasketch", + "expected": "h2.hashfunc)", + "got": "h2.hashfunc)", + "got_raw": "h2.hashfunc)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ekzhu/datasketch", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ekzhu/datasketch", + "expected": "4)", + "got": "1 << 4)", + "got_raw": "1 << 4)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "eliangcs/pystock-crawler", + "expected": "{ 'symbol': 'ALL', 'amend': False, 'doc_type': '10-Q', 'period_focus': 'Q1', 'fiscal_year': 2013, 'end_date': '2013-03-31', 'revenues': 8463000000, 'op_income': None, 'net_income': 709000000, 'eps_basic': 1.49, 'eps_diluted': 1.47, 'dividend': 0.25, 'assets': 126612000000, 'cur_assets': None, 'cur_liab': None, 'equity': 20619000000, 'cash': 820000000, 'cash_flow_op': 740000000, 'cash_flow_inv': 136000000, 'cash_flow_fin': -862000000 })", + "got": "'20130331')", + "got_raw": "'20130331')", + "exact_match": false, + "code_bleu": 0.012946625684620988, + "edit_similarity": 0.0044444444444444444 + }, + { + "repo": "eliangcs/pystock-crawler", + "expected": "{ 'symbol': 'VNO', 'amend': False, 'doc_type': '10-K', 'period_focus': 'FY', 'fiscal_year': 2011, 'end_date': '2011-12-31', 'revenues': 2915665000, 'op_income': 856153000, 'net_income': 601771000, 'eps_basic': 3.26, 'eps_diluted': 3.23, 'dividend': 0.0, 'assets': 20446487000, 'cur_assets': None, 'cur_liab': None, 'equity': 7508447000, 'cash': 606553000, 'cash_flow_op': 702499000, 'cash_flow_inv': -164761000, 'cash_flow_fin': -621974000 })", + "got": "'vno-20111231')", + "got_raw": "'vno-20111231')", + "exact_match": false, + "code_bleu": 0.0127924991883755, + "edit_similarity": 0.01312910284463895 + }, + { + "repo": "eliangcs/pystock-crawler", + "expected": "{ 'symbol': 'UTMD', 'amend': False, 'doc_type': '10-K', 'period_focus': 'FY', 'fiscal_year': 2011, 'end_date': '2011-12-31', 'revenues': 37860000, 'op_income': 11842000, 'net_income': 7414000, 'eps_basic': 2.04, 'eps_diluted': 2.03, 'dividend': 0.0, 'assets': 76389000, 'cur_assets': 17016000, 'cur_liab': 9631000, 'equity': 40757000, 'cash': 6534000, 'cash_flow_op': 11365000, 'cash_flow_inv': -26685000, 'cash_flow_fin': 18078000 })", + "got": "'utmd-20111231')", + "got_raw": "'utmd-20111231')", + "exact_match": false, + "code_bleu": 0.012946625684620988, + "edit_similarity": 0.013333333333333334 + }, + { + "repo": "eliangcs/pystock-crawler", + "expected": "{ 'symbol': 'BK', 'amend': False, 'doc_type': '10-Q', 'period_focus': 'Q1', 'fiscal_year': 2010, 'end_date': '2010-03-31', 'revenues': 883000000, 'op_income': None, 'net_income': 559000000, 'eps_basic': 0.46, 'eps_diluted': 0.46, 'dividend': 0.09, 'assets': 220551000000, 'cur_assets': None, 'cur_liab': None, 'equity': 30455000000, 'cash': 3307000000, 'cash_flow_op': 1191000000, 'cash_flow_inv': 512000000, 'cash_flow_fin': -2126000000 })", + "got": "'bk-20100331')", + "got_raw": "'bk-20100331')", + "exact_match": false, + "code_bleu": 0.012946625684620988, + "edit_similarity": 0.013215859030837005 + }, + { + "repo": "eliangcs/pystock-crawler", + "expected": "content)", + "got": "content)", + "got_raw": "content)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "eliangcs/pystock-crawler", + "expected": "{ 'symbol': 'PAY', 'amend': False, 'doc_type': '10-Q', 'period_focus': 'Q2', 'fiscal_year': 2011, 'end_date': '2011-04-30', 'revenues': 292446000, 'op_income': 37338000, 'net_income': 25200000, 'eps_basic': 0.29, 'eps_diluted': 0.27, 'dividend': 0.0, 'assets': 1252289000, 'cur_assets': 935395000, 'cur_liab': 303590000, 'equity': 332172000, 'cash': 531542000, 'cash_flow_op': 68831000, 'cash_flow_inv': -20049000, 'cash_flow_fin': 34676000 })", + "got": "'pay-20110430')", + "got_raw": "'pay-20110430')", + "exact_match": false, + "code_bleu": 0.012946625684620988, + "edit_similarity": 0.013100436681222707 + }, + { + "repo": "eliangcs/pystock-crawler", + "expected": "expected.get('cash'))", + "got": "expected.get('cash'))", + "got_raw": "expected.get('cash'))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "evansd/whitenoise", + "expected": "response.headers", + "got": "response", + "got_raw": "response", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "evansd/whitenoise", + "expected": "\"application/pkc7-mime\"", + "got": "\"application/json\"", + "got_raw": "\"application/json\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6829268292682927 + }, + { + "repo": "evansd/whitenoise", + "expected": "404", + "got": "404", + "got_raw": "404", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "evansd/whitenoise", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "evansd/whitenoise", + "expected": "(\"woops\",)", + "got": "(\"woops\",)", + "got_raw": "(\"woops\",)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "executablebooks/markdown-it-py", + "expected": "heading_node.map", + "got": "tuple(heading_node.map or ())", + "got_raw": "tuple(heading_node.map or ())", + "exact_match": false, + "code_bleu": 0.2626909894424158, + "edit_similarity": 0.7111111111111111 + }, + { + "repo": "executablebooks/markdown-it-py", + "expected": "99", + "got": "99", + "got_raw": "99", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "executablebooks/markdown-it-py", + "expected": "\"text\"", + "got": "\"text\"", + "got_raw": "\"text\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "executablebooks/markdown-it-py", + "expected": "heading_node.meta", + "got": "heading_node.meta", + "got_raw": "heading_node.meta", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "executablebooks/markdown-it-py", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "executablebooks/markdown-it-py", + "expected": "env", + "got": "env", + "got_raw": "env", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "executablebooks/markdown-it-py", + "expected": "SystemExit)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.2727272727272727 + }, + { + "repo": "exercism/python", + "expected": "[[4]])", + "got": "[[1]])", + "got_raw": "[[1]])", + "exact_match": false, + "code_bleu": 0.488923022434901, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "exercism/python", + "expected": "Rational(1, 3))", + "got": "Rational(1, 3))", + "got_raw": "Rational(1, 3))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "exercism/python", + "expected": "[8])", + "got": "8)", + "got_raw": "8)", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "exercism/python", + "expected": "[-1])", + "got": "[-1])", + "got_raw": "[-1])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "exercism/python", + "expected": "[1])", + "got": "[1])", + "got_raw": "[1])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "exercism/python", + "expected": "\"V\")", + "got": "'V')", + "got_raw": "'V')", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "exercism/python", + "expected": "6)", + "got": "6)", + "got_raw": "6)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "exercism/python", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "exercism/python", + "expected": "\"II\")", + "got": "\"II\")", + "got_raw": "\"II\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "exercism/python", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "exercism/python", + "expected": "[1])", + "got": "[1])", + "got_raw": "[1])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "exercism/python", + "expected": "True)", + "got": "True)", + "got_raw": "True)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "exercism/python", + "expected": "\"4\")", + "got": "\"4\")", + "got_raw": "\"4\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "exercism/python", + "expected": "[[5]])", + "got": "[[5]])", + "got_raw": "[[5]])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "exercism/python", + "expected": "3)", + "got": "3)", + "got_raw": "3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "exercism/python", + "expected": "[1])", + "got": "[1])", + "got_raw": "[1])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "exercism/python", + "expected": "robot.name)", + "got": "robot.name)", + "got_raw": "robot.name)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "exercism/python", + "expected": "date(2013, 3, 4))", + "got": "date(2013, 3, 11))", + "got_raw": "date(2013, 3, 11))", + "exact_match": false, + "code_bleu": 0.7071067811865475, + "edit_similarity": 0.9142857142857143 + }, + { + "repo": "exercism/python", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "exercism/python", + "expected": "queue)", + "got": "expected_result)", + "got_raw": "expected_result)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.2727272727272727 + }, + { + "repo": "explosion/sense2vec", + "expected": "s2v", + "got": "s2v.most_similar(\"beekeeper|NOUN\")", + "got_raw": "s2v.most_similar(\"beekeeper|NOUN\")", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.16216216216216217 + }, + { + "repo": "explosion/sense2vec", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "\"wrld\"", + "got": "\"wrld\"", + "got_raw": "\"wrld\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "s2v", + "got": "s2v", + "got_raw": "s2v", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/sense2vec", + "expected": "1.0", + "got": "1.0", + "got_raw": "1.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-llm", + "expected": "len(examples)", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/spacy-llm", + "expected": "ValueError, match=\"unexpected value; permitted\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.3728813559322034 + }, + { + "repo": "explosion/spacy-llm", + "expected": "[\"llm\"]", + "got": "[\"llm\"]", + "got_raw": "[\"llm\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-llm", + "expected": "2", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/spacy-llm", + "expected": "( [\"-PRON-\", \"have\", \"water\", \"the\", \"plant\", \".\"], [\"I\", \"have\", \"water\", \"the\", \"plant\", \".\"], )", + "got": "[[\"water\"], [\"water\", \"water\"]]", + "got_raw": "[[\"water\"], [\"water\", \"water\"]]", + "exact_match": false, + "code_bleu": 0.10817525998386816, + "edit_similarity": 0.34108527131782945 + }, + { + "repo": "explosion/spacy-llm", + "expected": "dict()", + "got": "dict()", + "got_raw": "dict()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-llm", + "expected": "gold_cats", + "got": "gold_cats", + "got_raw": "gold_cats", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-llm", + "expected": "f\"\"\" You are an expert summarization system. Your task is to accept Text as input and summarize the Text in a concise way. The summary must not, under any circumstances, contain more than 10 words. Here is the Text that needs to be summarized: ''' {example_text} ''' Summary:\"\"\".strip()", + "got": "prompt.strip()", + "got_raw": "prompt.strip()", + "exact_match": false, + "code_bleu": 0.47489730353224535, + "edit_similarity": 0.05333333333333334 + }, + { + "repo": "explosion/spacy-llm", + "expected": "[\"llm\"]", + "got": "[\"llm\"]", + "got_raw": "[\"llm\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-llm", + "expected": "n_prompt_examples", + "got": "n_prompt_examples", + "got_raw": "n_prompt_examples", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-llm", + "expected": "gold_ents", + "got": "gold_ents", + "got_raw": "gold_ents", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-llm", + "expected": "n_prompt_examples", + "got": "n_prompt_examples", + "got_raw": "n_prompt_examples", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-llm", + "expected": "labels", + "got": "labels", + "got_raw": "labels", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-llm", + "expected": "\"Q60\"", + "got": "llm._task._candidate_selector", + "got_raw": "llm._task._candidate_selector", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/spacy-llm", + "expected": "labels", + "got": "labels", + "got_raw": "labels", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-llm", + "expected": "ValueError, match=re.escape( \"Request to OpenAI API failed: This model's maximum context length is 4097 tokens. However, your messages \" \"resulted in 5018 tokens. Please reduce the length of the messages.\" ))", + "got": "pytest.raises(ValueError)", + "got_raw": "pytest.raises(ValueError)", + "exact_match": false, + "code_bleu": 0.13888095170058956, + "edit_similarity": 0.02575107296137339 + }, + { + "repo": "explosion/spacy-llm", + "expected": "gold_spans", + "got": "gold_spans", + "got_raw": "gold_spans", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/spacy-llm", + "expected": "ValueError, match=\"Request to Cohere API failed:\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.36065573770491804 + }, + { + "repo": "explosion/thinc", + "expected": "\"cool\"", + "got": "\"cool\"", + "got_raw": "\"cool\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "d_inputs[i])", + "got": "d_inputs[i])", + "got_raw": "d_inputs[i])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "ragged.lengths[start:end])", + "got": "ragged.lengths[start:end])", + "got_raw": "ragged.lengths[start:end])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "target_scores[0])", + "got": "target_scores[0])", + "got_raw": "target_scores[0])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "values3)", + "got": "values3)", + "got_raw": "values3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "[4, 6, 14, 15, 19]", + "got": "[4, 6, 14, 15, 16]", + "got_raw": "[4, 6, 14, 15, 16]", + "exact_match": false, + "code_bleu": 0.8265168183793802, + "edit_similarity": 0.9444444444444444 + }, + { + "repo": "explosion/thinc", + "expected": "ops.alloc2f(10, 2))", + "got": "asscalar(a)", + "got_raw": "asscalar(a)", + "exact_match": false, + "code_bleu": 0.17672682861490013, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "explosion/thinc", + "expected": "p_nonzero)", + "got": "ice(nonzero_empirical)", + "got_raw": "ice(nonzero_empirical)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.5 + }, + { + "repo": "explosion/thinc", + "expected": "list(Xs[0][0])", + "got": "[1]", + "got_raw": "[1]", + "exact_match": false, + "code_bleu": 0.14999999999999997, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "explosion/thinc", + "expected": "X.dataXd.shape", + "got": "X.dataXd.shape", + "got_raw": "X.dataXd.shape", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "(2, 45)", + "got": "(2, 45)", + "got_raw": "(2, 45)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "rate2", + "got": "1.0", + "got_raw": "1.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "explosion/thinc", + "expected": "\"linear\"", + "got": "\"linear\"", + "got_raw": "\"linear\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "[0, 4, 6, 14, 15]", + "got": "[0, 4, 6, 14, 15]", + "got_raw": "[0, 4, 6, 14, 15]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "Y.data.dtype", + "got": "Y.data.dtype", + "got_raw": "Y.data.dtype", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "123", + "got": "123", + "got_raw": "123", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "explosion/thinc", + "expected": "[[1.0], [1.0], [1.0]])", + "got": "0.0)", + "got_raw": "0.0)", + "exact_match": false, + "code_bleu": 0.076754995130253, + "edit_similarity": 0.23076923076923078 + }, + { + "repo": "explosion/thinc", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "eyurtsev/kor", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "eyurtsev/kor", + "expected": "exception_class)", + "got": "exception_class)", + "got_raw": "exception_class)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "eyurtsev/kor", + "expected": "[(\"input\", {\"key1\": \"value1\"})]", + "got": "BJECT_EXAMPLES[0]", + "got_raw": "BJECT_EXAMPLES[0]", + "exact_match": false, + "code_bleu": 0.14459467795764558, + "edit_similarity": 0.08333333333333333 + }, + { + "repo": "eyurtsev/kor", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "eyurtsev/kor", + "expected": "[ (\"another number\", {\"object\": [{\"number\": [1]}]}), (\"number\", {\"object\": [{\"number\": [2]}]}), (\"text\", {\"object\": [{\"text\": [\"3\"]}]}), (\"selection\", {\"object\": [{\"selection\": [\"option\"]}]}), (\"foo\", {}), (\"1 2\", {\"object\": [{\"age\": [1, 2]}]}), ]", + "got": "[{\"number\": 2, \"text\": \"3\", \"selection\": \"option\", \"age\": [1, 2]}]", + "got_raw": "[{\"number\": 2, \"text\": \"3\", \"selection\": \"option\", \"age\": [1, 2]}]", + "exact_match": false, + "code_bleu": 0.12975333076983192, + "edit_similarity": 0.28753993610223644 + }, + { + "repo": "eyurtsev/kor", + "expected": "extraction_subclass.__name__", + "got": "\"ExtractionSchemaNode\"", + "got_raw": "\"ExtractionSchemaNode\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.44 + }, + { + "repo": "eyurtsev/kor", + "expected": "[ (\"another number\", {\"object\": [{\"number\": 1}]}), (\"number\", {\"object\": [{\"number\": [2]}]}), (\"text\", {\"object\": [{\"text\": [\"3\"]}]}), (\"selection\", {\"object\": [{\"selection\": [\"option\"]}]}), (\"foo\", {}), ]", + "got": "[(\"number\", 2), (\"text\", \"3\"), (\"option\", \"selection\"), (\"foo\",)]", + "got_raw": "[(\"number\", 2), (\"text\", \"3\"), (\"option\", \"selection\"), (\"foo\",)]", + "exact_match": false, + "code_bleu": 0.14498692802436905, + "edit_similarity": 0.25925925925925924 + }, + { + "repo": "fabriziosalmi/certmate", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fabriziosalmi/certmate", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fabriziosalmi/certmate", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fabriziosalmi/certmate", + "expected": "data", + "got": "data", + "got_raw": "data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fabriziosalmi/certmate", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fabriziosalmi/certmate", + "expected": "msg", + "got": "msg.lower()", + "got_raw": "msg.lower()", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.42857142857142855 + }, + { + "repo": "facebookresearch/partnr-planner", + "expected": "63", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "facebookresearch/partnr-planner", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "facebookresearch/partnr-planner", + "expected": "5", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "facebookresearch/partnr-planner", + "expected": "gt_prompt", + "got": "gt_prompt", + "got_raw": "gt_prompt", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "facebookresearch/partnr-planner", + "expected": "TTT", + "got": "TTT", + "got_raw": "TTT", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "facebookresearch/partnr-planner", + "expected": "1.5", + "got": "base_pos", + "got_raw": " base_pos", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "facebookresearch/partnr-planner", + "expected": "\"\"", + "got": "dataset.episodes[0].scene_id", + "got_raw": "dataset.episodes[0].scene_id", + "exact_match": false, + "code_bleu": 0.13485111859503685, + "edit_similarity": 0.0 + }, + { + "repo": "facebookresearch/theseus", + "expected": "len(all_ids)", + "got": "len(all_ids)", + "got_raw": "len(all_ids)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "facebookresearch/theseus", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "facebookresearch/theseus", + "expected": "(batch_size, err_dim, i + 1)", + "got": "(batch_size, err_dim, i + 1)", + "got_raw": "(batch_size, err_dim, i + 1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "facebookresearch/theseus", + "expected": "len(all_ids)", + "got": "len(all_ids)", + "got_raw": "len(all_ids)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "facebookresearch/theseus", + "expected": "sol2[f\"x{i}\"])", + "got": "sol2[f\"x{i}\"])", + "got_raw": "sol2[f\"x{i}\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "facebookresearch/theseus", + "expected": "aux_vars", + "got": "optim_vars", + "got_raw": "optim_vars", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "facebookresearch/theseus", + "expected": "expected_jac[1].shape", + "got": "expected_jac[1].shape", + "got_raw": "expected_jac[1].shape", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "facebookresearch/theseus", + "expected": "default_order.index_of(var.name)", + "got": "default_order.get_index(var)", + "got_raw": "default_order.get_index(var)", + "exact_match": false, + "code_bleu": 0.2869522062208836, + "edit_similarity": 0.8 + }, + { + "repo": "facebookresearch/theseus", + "expected": "av)", + "got": "av)", + "got_raw": "av)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "facebookresearch/theseus", + "expected": "1", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "facebookresearch/theseus", + "expected": "q_inv_v", + "got": "q_inv_v", + "got_raw": "q_inv_v", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "facebookresearch/theseus", + "expected": "da_dx_unroll)", + "got": "da_dx_unroll)", + "got_raw": "da_dx_unroll)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "facebookresearch/theseus", + "expected": "1e-5", + "got": "1e-5", + "got_raw": "1e-5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "facebookresearch/theseus", + "expected": "expected_jac[0].shape", + "got": "expected_jac.shape", + "got_raw": "expected_jac.shape", + "exact_match": false, + "code_bleu": 0.3194715521231362, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "facebookresearch/theseus", + "expected": "data.shape", + "got": "(batch_size, length)", + "got_raw": "(batch_size, length)", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "facebookresearch/theseus", + "expected": "info.best_err)", + "got": "updated_inputs[\"theta\"])", + "got_raw": "updated_inputs[\"theta\"])", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.2631578947368421 + }, + { + "repo": "facebookresearch/theseus", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "facebookresearch/theseus", + "expected": "t1.norm()", + "got": "torch.norm(t1)", + "got_raw": "torch.norm(t1)", + "exact_match": false, + "code_bleu": 0.45499414040480374, + "edit_similarity": 0.6956521739130435 + }, + { + "repo": "facebookresearch/theseus", + "expected": "new_name", + "got": "new_name", + "got_raw": "new_name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi-mvc/fastapi-mvc", + "expected": "router2_expected", + "got": "router2_expected", + "got_raw": "router2_expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi-mvc/fastapi-mvc", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi-mvc/fastapi-mvc", + "expected": "caplog.text", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi-mvc/fastapi-mvc", + "expected": "[f\"{tmp}/my-script.sh\"])", + "got": "tmp)", + "got_raw": "tmp)", + "exact_match": false, + "code_bleu": 0.2149139863647084, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "fastapi-mvc/fastapi-mvc", + "expected": "[ mock.call(cmd=[\"git\", \"init\"], cwd=f\"{DIR}/test-project\"), mock.call(cmd=[\"make\", \"install\"], cwd=f\"{DIR}/test-project\"), ])", + "got": "\"fastapi_mvc.cli.new.copier\")", + "got_raw": "\"fastapi_mvc.cli.new.copier\")", + "exact_match": false, + "code_bleu": 0.027553075174962614, + "edit_similarity": 0.12903225806451613 + }, + { + "repo": "fastapi-mvc/fastapi-mvc", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi-mvc/fastapi-mvc", + "expected": "\"fake_project\"", + "got": "\"fake-project\"", + "got_raw": "\"fake-project\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9285714285714286 + }, + { + "repo": "fastapi-mvc/fastapi-mvc", + "expected": "dirhash(str(reference_projects[target]))", + "got": "dirhash(str(update_copy))", + "got_raw": "dirhash(str(update_copy))", + "exact_match": false, + "code_bleu": 0.4162224902509523, + "edit_similarity": 0.5230769230769231 + }, + { + "repo": "fastapi-users/fastapi-users", + "expected": "[\"200\", \"401\"]", + "got": "[status.HTTP_200_OK, status.HTTP_401_UNAUTHORIZED]", + "got_raw": "[status.HTTP_200_OK, status.HTTP_401_UNAUTHORIZED]", + "exact_match": false, + "code_bleu": 0.16784459625186196, + "edit_similarity": 0.3125 + }, + { + "repo": "fastapi-users/fastapi-users", + "expected": "status.HTTP_200_OK", + "got": "status.HTTP_200_OK", + "got_raw": "status.HTTP_200_OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi-users/fastapi-users", + "expected": "True", + "got": "False", + "got_raw": "False", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "fastapi-users/fastapi-users", + "expected": "[ \"200\", \"401\", \"403\", \"404\", \"400\", \"422\", ]", + "got": "[status.HTTP_200_OK, status.HTTP_404_NOT_FOUND]", + "got_raw": "[status.HTTP_200_OK, status.HTTP_404_NOT_FOUND]", + "exact_match": false, + "code_bleu": 0.10790009759048269, + "edit_similarity": 0.21739130434782608 + }, + { + "repo": "fastapi-users/fastapi-users", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi-users/fastapi-users", + "expected": "123", + "got": "123", + "got_raw": "123", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi-users/fastapi-users", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi-users/fastapi-users", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi-users/fastapi-users", + "expected": "\"/oauth/callback\"", + "got": "\"/oauth/callback\"", + "got_raw": "\"/oauth/callback\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi-users/fastapi-users", + "expected": "str(user.id)", + "got": "str(user.id)", + "got_raw": "str(user.id)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "heroes[1].name", + "got": "\"Deadpond\"", + "got_raw": "\"Deadpond\"", + "exact_match": false, + "code_bleu": 0.08333333333333333, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "captured.out", + "got": "captured.out", + "got_raw": "captured.out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "[ [{\"secret_name\": \"Dive Wilson\", \"age\": None, \"id\": 1, \"name\": \"Deadpond\"}] ]", + "got": "[", + "got_raw": "[\n \"Creating tables\",\n \"Creating tables (using declarative)\",\n \"Creating tables (using declarative) (using metadata)\",\n \"Creating tables (using declarative) (using metadata) (using metadata)\",\n \"Creating tables (using declarative) (using metadata) (using metadata) (using metadata)\",\n \"Creating tables (using declarative) (using metadata) (using metadata) (using metadata) (using metadata)\",\n \"Creating tables (using declarative) (using metadata) (using metadata) (using metadata) (using metadata) (using metadata)\",\n \"Creating tables (using declar", + "exact_match": false, + "code_bleu": 0.028314455119112403, + "edit_similarity": 0.02531645569620253 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "\"Dive Wilson\"", + "got": "\"Dive Wilson\"", + "got_raw": "\"Dive Wilson\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "expected_calls", + "got": "expected_calls", + "got_raw": "expected_calls", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "[ [ \"Hero:\", { \"id\": 1, \"name\": \"Test Hero\", \"secret_name\": \"Secret Test Hero\", \"age\": 24, }, ] ]", + "got": "[ \"SELECT hero.id, hero.name, hero.secret_name, hero.age FROM hero\" ]", + "got_raw": "[ \"SELECT hero.id, hero.name, hero.secret_name, hero.age FROM hero\" ]", + "exact_match": false, + "code_bleu": 0.05999999999999999, + "edit_similarity": 0.4939759036144578 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "RuntimeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "hero_2.id", + "got": "hero_2.id", + "got_raw": "hero_2.id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "\"CASCADE\"", + "got": "\"CASCADE\"", + "got_raw": "\"CASCADE\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "IntegrityError)", + "got": "IntegrityError)", + "got_raw": "IntegrityError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "\"Dive Wilson\"", + "got": "\"Dive Wilson\"", + "got_raw": "\"Dive Wilson\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "\"B\"", + "got": "\"A\"", + "got_raw": "\"A\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "str", + "got": "str", + "got_raw": "str", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fastapi/sqlmodel", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "feldroy/air", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "feldroy/air", + "expected": "html", + "got": "html", + "got_raw": "html", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "feldroy/air", + "expected": "response.text", + "got": "response.text", + "got_raw": "response.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "feldroy/air", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "feldroy/air", + "expected": "r.text", + "got": "r.text", + "got_raw": "r.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "feldroy/air", + "expected": "\"Bar\"", + "got": "\"Bar\"", + "got_raw": "\"Bar\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "feldroy/air", + "expected": "\"id_\"", + "got": "\"id_\"", + "got_raw": "\"id_\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "feldroy/air", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "feldroy/air", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "feldroy/air", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "feldroy/air", + "expected": "404", + "got": "404", + "got_raw": "404", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "feldroy/air", + "expected": "\"

Is boosted: True

\"", + "got": "\"Is boosted: True\"", + "got_raw": "\"Is boosted: True\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8 + }, + { + "repo": "feldroy/air", + "expected": "\"minified\"", + "got": "\"minified\"", + "got_raw": "\"minified\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "feldroy/air", + "expected": "expected_html", + "got": "expected_html", + "got_raw": "expected_html", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "feldroy/air", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "feldroy/air", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "feldroy/air", + "expected": "\"

Prompt: None

\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.16 + }, + { + "repo": "feldroy/air", + "expected": "response.text", + "got": "response.text", + "got_raw": "response.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "feldroy/air", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "feldroy/air", + "expected": "500", + "got": "500", + "got_raw": "500", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fennerm/flashfocus", + "expected": "a_copy", + "got": "a_copy", + "got_raw": "a_copy", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fennerm/flashfocus", + "expected": "list()", + "got": "windows[2:]", + "got_raw": "windows[2:]", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "fennerm/flashfocus", + "expected": "pytest.approx(opacity)", + "got": "opacity", + "got_raw": "opacity", + "exact_match": false, + "code_bleu": 0.09910059291689341, + "edit_similarity": 0.4827586206896552 + }, + { + "repo": "fennerm/flashfocus", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fennerm/flashfocus", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "{sm.wounded}", + "got": "first_peril", + "got_raw": "first_peril", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.17391304347826086 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "sm2.configuration_values", + "got": "sm2.configuration_values", + "got_raw": "sm2.configuration_values", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "\"yellow\"", + "got": "\"yellow\"", + "got_raw": "\"yellow\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "ValueError, match=\"A failed\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.55 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "vals", + "got": "vals", + "got_raw": "vals", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "1", + "got": "sm.custom", + "got_raw": "sm.custom", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "\"Stop\"", + "got": "\"Stop\"", + "got_raw": "\"Stop\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "sm2.model.name", + "got": "sm2.model.name", + "got_raw": "sm2.model.name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "\"hello\")", + "got": "\"hello\")", + "got_raw": "\"hello\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "\"42\"", + "got": "\"42\"", + "got_raw": "\"42\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "[(\"s2\", \"s3\")]", + "got": "[(\"s2\", \"s3\")]", + "got_raw": "[(\"s2\", \"s3\")]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "\"stopped\"", + "got": "\"stopped\"", + "got_raw": "\"stopped\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fgmacedo/python-statemachine", + "expected": "[4, 6]", + "got": "[4, 6]", + "got_raw": "[4, 6]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fixie-ai/ultravox", + "expected": "389", + "got": "960000", + "got_raw": "960000", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "fixie-ai/ultravox", + "expected": "[3000, 500]", + "got": "[35]", + "got_raw": "[35]", + "exact_match": false, + "code_bleu": 0.29999999999999993, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "fixie-ai/ultravox", + "expected": "pytest.approx(29.728070921986767, rel=1e-2)", + "got": "0.8333333333333334", + "got_raw": "0.8333333333333334", + "exact_match": false, + "code_bleu": 0.041666666666666664, + "edit_similarity": 0.06557377049180328 + }, + { + "repo": "fixie-ai/ultravox", + "expected": "processed", + "got": "processed", + "got_raw": "processed", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fixie-ai/ultravox", + "expected": "pytest.approx(100, rel=1e-2)", + "got": "0.5", + "got_raw": "0.5", + "exact_match": false, + "code_bleu": 0.041666666666666664, + "edit_similarity": 0.06451612903225806 + }, + { + "repo": "fixie-ai/ultravox", + "expected": "8", + "got": "8", + "got_raw": "8", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fixie-ai/ultravox", + "expected": "[ {\"role\": \"user\", \"content\": \"A\"}, {\"role\": \"assistant\", \"content\": \"C\"}, ]", + "got": "[{\"role\": \"user\", \"content\": \"A\"}]", + "got_raw": "[{\"role\": \"user\", \"content\": \"A\"}]", + "exact_match": false, + "code_bleu": 0.4617366309441026, + "edit_similarity": 0.6181818181818182 + }, + { + "repo": "fixie-ai/ultravox", + "expected": "torch.Size([15])", + "got": "(1, 15)", + "got_raw": "(1, 15)", + "exact_match": false, + "code_bleu": 0.20291744717193683, + "edit_similarity": 0.34782608695652173 + }, + { + "repo": "fixie-ai/ultravox", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fla-org/flash-linear-attention", + "expected": "1e-3)", + "got": "0.005)", + "got_raw": "0.005)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "fla-org/flash-linear-attention", + "expected": "0.006)", + "got": "0.005)", + "got_raw": "0.005)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "fla-org/flash-linear-attention", + "expected": "0.008)", + "got": "0.008)", + "got_raw": "0.008)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fla-org/flash-linear-attention", + "expected": "0.001)", + "got": "0.005)", + "got_raw": "0.005)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "fla-org/flash-linear-attention", + "expected": "1e-4)", + "got": "0.005)", + "got_raw": "0.005)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "fla-org/flash-linear-attention", + "expected": "xw1)", + "got": "xw1)", + "got_raw": "xw1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fla-org/flash-linear-attention", + "expected": "0.008)", + "got": "0.005)", + "got_raw": "0.005)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "fla-org/flash-linear-attention", + "expected": "0.008)", + "got": "0.01)", + "got_raw": "0.01)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "fla-org/flash-linear-attention", + "expected": "opt_split_none)", + "got": "ref_split_none)", + "got_raw": "ref_split_none)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8 + }, + { + "repo": "fla-org/flash-linear-attention", + "expected": "0.005)", + "got": "0.005)", + "got_raw": "0.005)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fla-org/flash-linear-attention", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fla-org/flash-linear-attention", + "expected": "0.007)", + "got": "0.007)", + "got_raw": "0.007)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fla-org/flash-linear-attention", + "expected": "0.004)", + "got": "0.005)", + "got_raw": "0.005)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "fla-org/flash-linear-attention", + "expected": "1e-3", + "got": "1e-3", + "got_raw": "1e-3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fla-org/flash-linear-attention", + "expected": "0.001)", + "got": "0.005)", + "got_raw": "0.005)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "fla-org/flash-linear-attention", + "expected": "cp)", + "got": "cp)", + "got_raw": "cp)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "flashbots/mev-inspect-py", + "expected": "arb_swaps", + "got": "arb_swaps", + "got_raw": "arb_swaps", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "flashbots/mev-inspect-py", + "expected": "\"0x4a536c1f6a5d5a9c1aeca9f6d04fbbf5f0d8f4e3\"", + "got": "\"0x4a536c1f6a5d5a9c1aeca9f6d04fbbf5f0d8f4e3\"", + "got_raw": "\"0x4a536c1f6a5d5a9c1aeca9f6d04fbbf5f0d8f4e3\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "flashbots/mev-inspect-py", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "flashbots/mev-inspect-py", + "expected": "\"testFunction(uint256)\"", + "got": "test_function_selector", + "got_raw": "test_function_selector", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "flashbots/mev-inspect-py", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "fpgmaas/deptry", + "expected": "{\"foo\": [Location(random_file, 1, 8)]}", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.08494336535733721, + "edit_similarity": 0.1 + }, + { + "repo": "fpgmaas/deptry", + "expected": "{ \"exclude\": [\"foo\", \"bar\"], \"extend_exclude\": [\"bar\", \"foo\"], \"ignore_notebooks\": True, \"per_rule_ignores\": { \"DEP001\": [\"baz\", \"foobar\"], \"DEP002\": [\"foo\"], \"DEP003\": [\"foobaz\"], \"DEP004\": [\"barfoo\"], }, \"ignore\": [\"DEP001\", \"DEP002\", \"DEP003\", \"DEP004\"], \"requirements_files\": \"foo.txt\", \"requirements_files_dev\": [\"dev.txt\", \"tests.txt\"], }", + "got": "{ \"exclude\": [\"foo\", \"bar\"], \"extend_exclude\": [\"bar\", \"foo\"], \"per_rule_ignores\": {\"DEP002\": [\"baz\", \"bar\"]}, \"ignore\": [], \"requirements_files\": \"requirements.txt\", \"requirements_files_dev\": [\"requirements-dev.txt\"], }", + "got_raw": "{ \"exclude\": [\"foo\", \"bar\"], \"extend_exclude\": [\"bar\", \"foo\"], \"per_rule_ignores\": {\"DEP002\": [\"baz\", \"bar\"]}, \"ignore\": [], \"requirements_files\": \"requirements.txt\", \"requirements_files_dev\": [\"requirements-dev.txt\"], }", + "exact_match": false, + "code_bleu": 0.40935111858044904, + "edit_similarity": 0.6347517730496454 + }, + { + "repo": "fpgmaas/deptry", + "expected": "{\"foo\", \"bar\"}", + "got": "[\"foo\", \"bar\"]", + "got_raw": "[\"foo\", \"bar\"]", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "fpgmaas/deptry", + "expected": "[ \"\", stylize(\"{BOLD}{GREEN}Success! No dependency issues found.{RESET}\"), ]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.1321341238891912, + "edit_similarity": 0.05128205128205128 + }, + { + "repo": "fpgmaas/deptry", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "fpgmaas/deptry", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fpgmaas/deptry", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fpgmaas/deptry", + "expected": "caplog.text", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fpgmaas/deptry", + "expected": "{\"click\"}", + "got": "[\"click\"]", + "got_raw": "[\"click\"]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "fpgmaas/deptry", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fpgmaas/deptry", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fpgmaas/deptry", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fronzbot/blinkpy", + "expected": "5678)", + "got": "5678)", + "got_raw": "5678)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fronzbot/blinkpy", + "expected": "record2)", + "got": "record2)", + "got_raw": "record2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fronzbot/blinkpy", + "expected": "await lotus.start())", + "got": "lotus.start())", + "got_raw": "lotus.start())", + "exact_match": false, + "code_bleu": 0.8571428571428571, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "fronzbot/blinkpy", + "expected": "\"test\")", + "got": "\"test\")", + "got_raw": "\"test\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fronzbot/blinkpy", + "expected": "\"refresh_456\"", + "got": "\"refresh_456\"", + "got_raw": "\"refresh_456\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fronzbot/blinkpy", + "expected": "None)", + "got": "\"unavailable\")", + "got_raw": "\"unavailable\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3157894736842105 + }, + { + "repo": "fronzbot/blinkpy", + "expected": "\"arm\")", + "got": "\"arm\")", + "got_raw": "\"arm\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fronzbot/blinkpy", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fronzbot/blinkpy", + "expected": "None)", + "got": "", + "got_raw": "", + "exact_match": false, + "code_bleu": 0.0, + "edit_similarity": 0.0 + }, + { + "repo": "fronzbot/blinkpy", + "expected": "\"token_123\"", + "got": "{\"access_token\": \"token_123\", \"refresh_token\": \"refresh_456\"}", + "got_raw": "{\"access_token\": \"token_123\", \"refresh_token\": \"refresh_456\"}", + "exact_match": false, + "code_bleu": 0.14113991930789777, + "edit_similarity": 0.3055555555555556 + }, + { + "repo": "fronzbot/blinkpy", + "expected": "[\"1\"])", + "got": "[\"1\"])", + "got_raw": "[\"1\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fronzbot/blinkpy", + "expected": "mock_server)", + "got": "COMMAND_DONE)", + "got_raw": "COMMAND_DONE)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.16 + }, + { + "repo": "fronzbot/blinkpy", + "expected": "await self.blink.set_status())", + "got": "await self.blink.set_status())", + "got_raw": "await self.blink.set_status())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fronzbot/blinkpy", + "expected": "1234)", + "got": "1234)", + "got_raw": "1234)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fronzbot/blinkpy", + "expected": "\"host\")", + "got": "\"host\")", + "got_raw": "\"host\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fudan-zvg/SETR", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fudan-zvg/SETR", + "expected": "(750, 1333, 3)", + "got": "(800, 1333, 3)", + "got_raw": "(800, 1333, 3)", + "exact_match": false, + "code_bleu": 0.7071067811865475, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "fudan-zvg/SETR", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fudan-zvg/SETR", + "expected": "4", + "got": "5", + "got_raw": "5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "fudan-zvg/SETR", + "expected": "get_classes('voc')", + "got": "get_classes('voc')", + "got_raw": "get_classes('voc')", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fudan-zvg/SETR", + "expected": "1200", + "got": "800", + "got_raw": "800", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "fudan-zvg/SETR", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fudan-zvg/SETR", + "expected": "64", + "got": "64", + "got_raw": "64", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "fudan-zvg/SETR", + "expected": "decode_head_cfg.num_classes", + "got": "decode_head_cfg.channels", + "got_raw": "decode_head_cfg.channels", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7843137254901961 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "\"default\"", + "got": "\"default\"", + "got_raw": "\"default\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "my_name", + "got": "my_name", + "got_raw": "my_name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "repr(platform_variant_manifest.architecture)", + "got": "platform_variant_manifest.os.architecture", + "got_raw": "platform_variant_manifest.os.architecture", + "exact_match": false, + "code_bleu": 0.356072505330195, + "edit_similarity": 0.8941176470588236 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "my_secret2", + "got": "my_secret2", + "got_raw": "my_secret2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "\"linux\"", + "got": "\"linux\"", + "got_raw": "\"linux\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "[docker_client.context.inspect(\"default\")]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.09090909090909091 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "repr(nodes)", + "got": "nodes[0].addresses", + "got_raw": "nodes[0].addresses", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.3448275862068966 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "8", + "got": "8", + "got_raw": "8", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gabrieldemarmiesse/python-on-whales", + "expected": "str(exception)", + "got": "str(exception)", + "got_raw": "str(exception)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gadievron/raptor", + "expected": "id2", + "got": "id2", + "got_raw": "id2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gadievron/raptor", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gadievron/raptor", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gadievron/raptor", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gadievron/raptor", + "expected": "16", + "got": "4", + "got_raw": "4", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "gadievron/raptor", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gadievron/raptor", + "expected": "missing", + "got": "missing", + "got_raw": "missing", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gadievron/raptor", + "expected": "0x1d8698", + "got": "0x1d8698", + "got_raw": "0x1d8698", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gadievron/raptor", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gadievron/raptor", + "expected": "response2.content", + "got": "response2.content", + "got_raw": "response2.content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gadievron/raptor", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gadievron/raptor", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gadievron/raptor", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gadievron/raptor", + "expected": "20", + "got": "20", + "got_raw": "20", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gadievron/raptor", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gadievron/raptor", + "expected": "rg", + "got": "rg", + "got_raw": "rg", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gadievron/raptor", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gao-lab/GLUE", + "expected": "logger2", + "got": "logger2", + "got_raw": "logger2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gao-lab/GLUE", + "expected": "foscttm_x.mean()", + "got": "foscttm_x", + "got_raw": "foscttm_x", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.72 + }, + { + "repo": "gao-lab/GLUE", + "expected": "4", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "gao-lab/GLUE", + "expected": "(atac.shape[0], 2)", + "got": "(atac.shape[0], 2)", + "got_raw": "(atac.shape[0], 2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gee-community/geetools", + "expected": "66", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "gee-community/geetools", + "expected": "[\"1\", \"2\", \"3\"]", + "got": "[\"1\", \"2\", \"3\"]", + "got_raw": "[\"1\", \"2\", \"3\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gee-community/geetools", + "expected": "1234.0", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "gee-community/geetools", + "expected": "[[[0, 1], [0, 0]]]", + "got": "[[[0, 0], [1, 1]], [[2, 2], [3, 3]]]", + "got_raw": "[[[0, 0], [1, 1]], [[2, 2], [3, 3]]]", + "exact_match": false, + "code_bleu": 0.3398611473701986, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "gee-community/geetools", + "expected": "[\"B1\", \"B2\", \"B3\", \"B1_1\", \"B1_2\"]", + "got": "[\"B1\", \"B1\"]", + "got_raw": "[\"B1\", \"B1\"]", + "exact_match": false, + "code_bleu": 0.24422043905080498, + "edit_similarity": 0.5217391304347826 + }, + { + "repo": "gee-community/geetools", + "expected": "{\"foo\": 1, \"bar\": 2}", + "got": "{\"foo\": 1, \"bar\": 2}", + "got_raw": "{\"foo\": 1, \"bar\": 2}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gee-community/geetools", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gee-community/geetools", + "expected": "\"2020-01-01\"", + "got": "\"2020-01-01\"", + "got_raw": "\"2020-01-01\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gee-community/geetools", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gee-community/geetools", + "expected": "ValueError)", + "got": "ee.EEException)", + "got_raw": "ee.EEException)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "geopy/geopy", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "geopy/geopy", + "expected": "location.address", + "got": "location.address", + "got_raw": "location.address", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "geopy/geopy", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "geopy/geopy", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "geopy/geopy", + "expected": "3", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "geopy/geopy", + "expected": "'country:FR'", + "got": "'country:FR'", + "got_raw": "'country:FR'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "geopy/geopy", + "expected": "ssl_context", + "got": "ssl_context", + "got_raw": "ssl_context", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "geopy/geopy", + "expected": "GeocoderAuthenticationFailure)", + "got": "GeocoderAuthenticationFailure)", + "got_raw": "GeocoderAuthenticationFailure)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "geopy/geopy", + "expected": "'my_user_agent/1.0'", + "got": "'my_user_agent/1.0'", + "got_raw": "'my_user_agent/1.0'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "geopy/geopy", + "expected": "GeocoderQueryError)", + "got": "GeocoderQueryError)", + "got_raw": "GeocoderQueryError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "geopy/geopy", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "geopy/geopy", + "expected": "allowed", + "got": "allowed", + "got_raw": "allowed", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "geopy/geopy", + "expected": "''", + "got": "''", + "got_raw": "''", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "geopy/geopy", + "expected": "12", + "got": "12", + "got_raw": "12", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "geopy/geopy", + "expected": "current_all", + "got": "current_all", + "got_raw": "current_all", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "geopy/geopy", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "geopy/geopy", + "expected": "d2", + "got": "d2", + "got_raw": "d2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "geopy/geopy", + "expected": "location.address", + "got": "location.address", + "got_raw": "location.address", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "geopy/geopy", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "geopy/geopy", + "expected": "res.address", + "got": "res.raw", + "got_raw": "res.raw", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "giampaolo/pyftpdlib", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "giampaolo/pyftpdlib", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "giampaolo/pyftpdlib", + "expected": "root", + "got": "root", + "got_raw": "root", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "giampaolo/pyftpdlib", + "expected": "\"Goodbye.\"", + "got": "\"Goodbye.\"", + "got_raw": "\"Goodbye.\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "giampaolo/pyftpdlib", + "expected": "\"/\"", + "got": "\"/\"", + "got_raw": "\"/\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "giampaolo/pyftpdlib", + "expected": "dir", + "got": "dir", + "got_raw": "dir", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "len(members_usernames_before) + 1", + "got": "len(members_before) + 1", + "got_raw": "len(members_before) + 1", + "exact_match": false, + "code_bleu": 0.488923022434901, + "edit_similarity": 0.8214285714285714 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "SystemExit)", + "got": "Exception)", + "got_raw": "Exception)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.38095238095238093 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "\"", + "got": "\"", + "got_raw": "\"#FF0000\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "group_name)", + "got": "group_name)", + "got_raw": "group_name)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "\"beta\"", + "got": "\"beta\"", + "got_raw": "\"beta\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "AccessLevel.OWNER.value", + "got": "AccessLevel.OWNER.value", + "got_raw": "AccessLevel.OWNER.value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "third_hook_before_test.asdict()", + "got": "third_hook_before_test.asdict()", + "got_raw": "third_hook_before_test.asdict()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "1", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "gitlabform/gitlabform", + "expected": "{\"foo\": True}", + "got": "{\"foo\": True}", + "got_raw": "{\"foo\": True}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gotalab/skillport", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gotalab/skillport", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gotalab/skillport", + "expected": "\"utf-8\"", + "got": "\"utf-8\"", + "got_raw": "\"utf-8\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gotalab/skillport", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gotalab/skillport", + "expected": "Exception)", + "got": "AttributeError)", + "got_raw": "AttributeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.32 + }, + { + "repo": "gotalab/skillport", + "expected": "\"application/octet-stream\"", + "got": "\"application/octet-stream\"", + "got_raw": "\"application/octet-stream\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gotalab/skillport", + "expected": "text", + "got": "text", + "got_raw": "text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gotalab/skillport", + "expected": "0.2", + "got": "0.2", + "got_raw": "0.2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gotalab/skillport", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gotalab/skillport", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gotalab/skillport", + "expected": "\"DEBUG\"", + "got": "\"DEBUG\"", + "got_raw": "\"DEBUG\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gotalab/skillport", + "expected": "\"utf-8\"", + "got": "\"utf-8\"", + "got_raw": "\"utf-8\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gotalab/skillport", + "expected": "\"skill-b\"", + "got": "\"skill-b\"", + "got_raw": "\"skill-b\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gotalab/skillport", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gouline/dbt-metabase", + "expected": "\"(start_date) - cast((rolling_days))\"", + "got": "\"Depends on\\n\\nQuestion 2!\"", + "got_raw": "\"Depends on\\n\\nQuestion 2!\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.1875 + }, + { + "repo": "gouline/dbt-metabase", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gouline/dbt-metabase", + "expected": "tables", + "got": "tables", + "got_raw": "tables", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gouline/dbt-metabase", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/daggr", + "expected": "\"HELLO\"", + "got": "\"HELLO\"", + "got_raw": "\"HELLO\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/daggr", + "expected": "choice", + "got": "choice", + "got_raw": "choice", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/daggr", + "expected": "8", + "got": "8", + "got_raw": "8", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/daggr", + "expected": "\"x\"", + "got": "\"x\"", + "got_raw": "\"x\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/daggr", + "expected": "\"step_a\"", + "got": "\"step_a\"", + "got_raw": "\"step_a\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/daggr", + "expected": "\"items\"", + "got": "\"items\"", + "got_raw": "\"items\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gradio-app/daggr", + "expected": "40", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "grapeot/devin.cursorrules", + "expected": "self.mock_openai_client)", + "got": "self.mock_openai_client)", + "got_raw": "self.mock_openai_client)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grapeot/devin.cursorrules", + "expected": "\"Test Gemini response\")", + "got": "\"Test Gemini response\")", + "got_raw": "\"Test Gemini response\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grapeot/devin.cursorrules", + "expected": "result)", + "got": "result)", + "got_raw": "result)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "grapeot/devin.cursorrules", + "expected": "mock_genai)", + "got": "mock_genai.GenerativeModel)", + "got_raw": "mock_genai.GenerativeModel)", + "exact_match": false, + "code_bleu": 0.3976353643835253, + "edit_similarity": 0.5789473684210527 + }, + { + "repo": "graphql-python/gql", + "expected": "\"AF\"", + "got": "\"AF\"", + "got_raw": "\"AF\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/gql", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/gql", + "expected": "TransportQueryError)", + "got": "TransportQueryError)", + "got_raw": "TransportQueryError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/gql", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/gql", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/gql", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/gql", + "expected": "\"val1\"", + "got": "\"value1\"", + "got_raw": "\"value1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "graphql-python/gql", + "expected": "\"test1234\"", + "got": "\"dummy\"", + "got_raw": "\"dummy\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "graphql-python/gql", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/gql", + "expected": "\"...DEF\"", + "got": "\"...DEF\"", + "got_raw": "\"...DEF\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/gql", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/gql", + "expected": "\"doc\"", + "got": "\"subscription:ok\"", + "got_raw": "\"subscription:ok\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2727272727272727 + }, + { + "repo": "graphql-python/gql", + "expected": "request_3.document", + "got": "request_3.document", + "got_raw": "request_3.document", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/gql", + "expected": "\"AF\"", + "got": "\"AF\"", + "got_raw": "\"AF\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/gql", + "expected": "\"3600\"", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "graphql-python/gql", + "expected": "TransportQueryError)", + "got": "TransportQueryError)", + "got_raw": "TransportQueryError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/gql", + "expected": "\"AF\"", + "got": "\"AF\"", + "got_raw": "\"AF\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/gql", + "expected": "captured_err", + "got": "captured_err", + "got_raw": "captured_err", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/gql", + "expected": "\"Europe\"", + "got": "\"Europe\"", + "got_raw": "\"Europe\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/gql", + "expected": "spread2", + "got": "spread2", + "got_raw": "spread2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene", + "expected": "Color", + "got": "Color", + "got_raw": "Color", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene", + "expected": "[MyFirstInterface]", + "got": "[MyFirstInterface]", + "got_raw": "[MyFirstInterface]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene", + "expected": "{\"optional\": {\"bool\": None}}", + "got": "{\"optional\": {\"bool\": None}}", + "got_raw": "{\"optional\": {\"bool\": None}}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene", + "expected": "Node", + "got": "Node", + "got_raw": "Node", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene", + "expected": "two", + "got": "two", + "got_raw": "two", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene", + "expected": "\"a\"", + "got": "\"a\"", + "got_raw": "\"a\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene", + "expected": "\"value\"", + "got": "\"value\"", + "got_raw": "\"value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene", + "expected": "\"Desc\"", + "got": "\"Desc\"", + "got_raw": "\"Desc\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene", + "expected": "[\"what\", \"client_mutation_id\"]", + "got": "[\"message\"]", + "got_raw": "[\"message\"]", + "exact_match": false, + "code_bleu": 0.29999999999999993, + "edit_similarity": 0.2926829268292683 + }, + { + "repo": "graphql-python/graphene", + "expected": "MyType", + "got": "MyType", + "got_raw": "MyType", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene", + "expected": "\"PageInfo\"", + "got": "\"PageInfo\"", + "got_raw": "\"PageInfo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene", + "expected": "Query", + "got": "Query", + "got_raw": "Query", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene", + "expected": "\"b\"", + "got": "\"b\"", + "got_raw": "\"b\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene", + "expected": "{\"say\": {\"phrase\": \"hello\"}}", + "got": "{'say': {'phrase': 'hello'}}", + "got_raw": "{'say': {'phrase': 'hello'}}", + "exact_match": false, + "code_bleu": 0.25406637407730737, + "edit_similarity": 0.7857142857142857 + }, + { + "repo": "graphql-python/graphene", + "expected": "String", + "got": "String", + "got_raw": "String", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene", + "expected": "{\"json\": json_value}", + "got": "{\"json\": json_value}", + "got_raw": "{\"json\": json_value}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene", + "expected": "\"1\"", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene", + "expected": "{\"node\": None}", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.23784142300054423, + "edit_similarity": 0.25 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "DOCUMENTARY", + "got": "DOCUMENTARY", + "got_raw": "DOCUMENTARY", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "[ {\"node\": {\"headline\": \"A\"}}, {\"node\": {\"headline\": \"AB\"}}, {\"node\": {\"headline\": \"B\"}}, ]", + "got": "[{\"node\": {\"headline\": \"A\"}}, {\"node\": {\"headline\": \"B\"}}]", + "got_raw": "[{\"node\": {\"headline\": \"A\"}}, {\"node\": {\"headline\": \"B\"}}]", + "exact_match": false, + "code_bleu": 0.6314407694409685, + "edit_similarity": 0.7785234899328859 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "DateTime)", + "got": "DateTime)", + "got_raw": "DateTime)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "Boolean)", + "got": "Boolean)", + "got_raw": "Boolean)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "body.items()", + "got": "body[\"query\"]", + "got_raw": "body[\"query\"]", + "exact_match": false, + "code_bleu": 0.28744328957046744, + "edit_similarity": 0.4 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "ReporterModel", + "got": "ReporterModel", + "got_raw": "ReporterModel", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "\"English\"", + "got": "\"English\"", + "got_raw": "\"English\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "UUID)", + "got": "UUID)", + "got_raw": "UUID)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "\"Hello\"", + "got": "\"Hello\"", + "got_raw": "\"Hello\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "graphene.Int", + "got": "Reporter.a_choice", + "got_raw": "Reporter.a_choice", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.13793103448275862 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "[ {\"node\": {\"headline\": \"A\"}}, ]", + "got": "[{\"node\": {\"headline\": \"A\"}}]", + "got_raw": "[{\"node\": {\"headline\": \"A\"}}]", + "exact_match": false, + "code_bleu": 0.8465171567308547, + "edit_similarity": 0.9508196721311475 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "\"Narf\"", + "got": "\"Narf\"", + "got_raw": "\"Narf\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "graphql-python/graphene-django", + "expected": "result.data[\"_debug\"][\"sql\"][0][\"rawSql\"]", + "got": "result.data[\"_debug\"][\"sql\"][0]", + "got_raw": "result.data[\"_debug\"][\"sql\"][0]", + "exact_match": false, + "code_bleu": 0.8, + "edit_similarity": 0.8611111111111112 + }, + { + "repo": "greyli/helloflask", + "expected": "302)", + "got": "302)", + "got_raw": "302)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "greyli/helloflask", + "expected": "200)", + "got": "200)", + "got_raw": "200)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "greyli/helloflask", + "expected": "200)", + "got": "200)", + "got_raw": "200)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "greyli/helloflask", + "expected": "self.app)", + "got": "self.app)", + "got_raw": "self.app)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "greyli/helloflask", + "expected": "403)", + "got": "403)", + "got_raw": "403)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "greyli/helloflask", + "expected": "app is None)", + "got": "app is None)", + "got_raw": "app is None)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "fake", + "got": "fake", + "got_raw": "fake", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "user_id", + "got": "user_id", + "got_raw": "user_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "whoami", + "got": "whoami", + "got_raw": "whoami", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "EQ(1, 1)", + "got": "1 == 1", + "got_raw": "1 == 1", + "exact_match": false, + "code_bleu": 0.26509024292235017, + "edit_similarity": 0.42857142857142855 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "[value])", + "got": "[value])", + "got_raw": "[value])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "\"\"", + "got": "links[0][\"Name\"]", + "got_raw": "links[0][\"Name\"]", + "exact_match": false, + "code_bleu": 0.1561969968460128, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "*args)", + "got": "*args)", + "got_raw": "*args)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "{ \"id\": \"rec123\", \"createdTime\": ANY, \"fields\": {\"Name\": \"Alice\"}, }", + "got": "{ \"id\": \"rec123\", \"createdTime\": ANY, \"fields\": {\"Name\": \"Alice\"}, }", + "got_raw": "{ \"id\": \"rec123\", \"createdTime\": ANY, \"fields\": {\"Name\": \"Alice\"}, }", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "value)", + "got": "value)", + "got_raw": "value)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "TypeError)", + "got": "pydantic.ValidationError)", + "got_raw": "pydantic.ValidationError)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.4 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "\"asdf\"", + "got": "\"asdf\"", + "got_raw": "\"asdf\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "[rec]", + "got": "[rec]", + "got_raw": "[rec]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "id_a", + "got": "id_a", + "got_raw": "id_a", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "[value, -1])", + "got": "[value, \"invalid\"])", + "got_raw": "[value, \"invalid\"])", + "exact_match": false, + "code_bleu": 0.41907687637277224, + "edit_similarity": 0.6451612903225806 + }, + { + "repo": "gtalarico/pyairtable", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hacs/integration", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hacs/integration", + "expected": "\"running\"", + "got": "\"setup\"", + "got_raw": "\"setup\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.375 + }, + { + "repo": "hacs/integration", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hacs/integration", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "hacs/integration", + "expected": "f\"{category_test_data['repository'] }/test_update_repository_entity.json\")", + "got": "hacs.repositories.get_by_full_name(category_test_data[\"repository\"])", + "got_raw": "hacs.repositories.get_by_full_name(category_test_data[\"repository\"])", + "exact_match": false, + "code_bleu": 0.11390778025531027, + "edit_similarity": 0.4507042253521127 + }, + { + "repo": "hacs/integration", + "expected": "caplog.text", + "got": "\"Store already exists\" in caplog.text", + "got_raw": "\"Store already exists\" in caplog.text", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.4583333333333333 + }, + { + "repo": "hacs/integration", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hacs/integration", + "expected": "\"default\"", + "got": "\"stable\"", + "got_raw": "\"stable\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "hacs/integration", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hacs/integration", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hacs/integration", + "expected": "HomeAssistantError, match=re.escape(\"This version requires HACS 9999.99.99 or newer.\"))", + "got": "HomeAssistantError)", + "got_raw": "HomeAssistantError)", + "exact_match": false, + "code_bleu": 0.1081097377275201, + "edit_similarity": 0.3584905660377358 + }, + { + "repo": "hacs/integration", + "expected": "\"user/repo\"", + "got": "\"user/repo\"", + "got_raw": "\"user/repo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hacs/integration", + "expected": "prev_url", + "got": "prev_url", + "got_raw": "prev_url", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hacs/integration", + "expected": "f\"{category_test_data['repository']}/test_update_entity_state.json\")", + "got": "f = snapshots.get_snapshot(category_test_data[\"category\"])", + "got_raw": "f = snapshots.get_snapshot(category_test_data[\"category\"])", + "exact_match": false, + "code_bleu": 0.12605968092174913, + "edit_similarity": 0.4126984126984127 + }, + { + "repo": "hacs/integration", + "expected": "HomeAssistantError, match=re.escape( \"No update available for update.basic_integration_update\", ))", + "got": "patch(\"homeassistant.components.hacs.update.async_update\")", + "got_raw": "patch(\"homeassistant.components.hacs.update.async_update\")", + "exact_match": false, + "code_bleu": 0.13254512146117509, + "edit_similarity": 0.44871794871794873 + }, + { + "repo": "hacs/integration", + "expected": "FlowResultType.FORM", + "got": "FlowResultType.FORM", + "got_raw": "FlowResultType.FORM", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hbldh/bleak", + "expected": "b\"1234\"", + "got": "b\"1234\"", + "got_raw": "b\"1234\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hbldh/bleak", + "expected": "NOTIFY_CHAR_UUID", + "got": "notify_characteristic.uuid.lower()", + "got_raw": "notify_characteristic.uuid.lower()", + "exact_match": false, + "code_bleu": 0.1561969968460128, + "edit_similarity": 0.04 + }, + { + "repo": "hbldh/bleak", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hbldh/bleak", + "expected": "b\"ind1\"", + "got": "b\"ind1\"", + "got_raw": "b\"ind1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hbldh/bleak", + "expected": "\"00000001-0000-1000-8000-00805f9b34fb\"", + "got": "\"00000001-0000-1000-8000-00805f9b34fb\"", + "got_raw": "\"00000001-0000-1000-8000-00805f9b34fb\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hbldh/bleak", + "expected": "INDICATE_CHAR_UUID", + "got": "INDICATE_CHAR_UUID", + "got_raw": "INDICATE_CHAR_UUID", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hbldh/bleak", + "expected": "\"BleakScannerWinRT\"", + "got": "\"BleakScannerWinRT\"", + "got_raw": "\"BleakScannerWinRT\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "dump[\"hd\"]", + "got": "dump[\"hd\"]", + "got_raw": "dump[\"hd\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "\"\"", + "got": "json.dumps(final_dumps[\"derivations\"][-1], indent=4, ensure_ascii=False)", + "got_raw": "json.dumps(final_dumps[\"derivations\"][-1], indent=4, ensure_ascii=False)", + "exact_match": false, + "code_bleu": 0.048853266442119285, + "edit_similarity": 0.05405405405405406 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "dump[\"hd\"]", + "got": "dump[\"hd\"]", + "got_raw": "dump[\"hd\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "dump[\"hd\"]", + "got": "dump[\"hd\"]", + "got_raw": "dump[\"hd\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "dump[\"hd\"]", + "got": "dump[\"hd\"]", + "got_raw": "dump[\"hd\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "dump[\"seed\"]", + "got": "dump[\"seed\"]", + "got_raw": "dump[\"seed\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "dump[\"ecc\"]", + "got": "dump[\"ecc\"]", + "got_raw": "dump[\"ecc\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "dump[\"ecc\"]", + "got": "dump[\"ecc\"]", + "got_raw": "dump[\"ecc\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "point_radd.y()", + "got": "point_radd.y()", + "got_raw": "point_radd.y()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "dump[\"ecc\"]", + "got": "dump[\"ecc\"]", + "got_raw": "dump[\"ecc\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "dump[\"hd\"]", + "got": "dump[\"hd\"]", + "got_raw": "dump[\"hd\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "__[\"name\"]", + "got": "__[\"name\"]", + "got_raw": "__[\"name\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "dump[\"seed\"]", + "got": "dump[\"seed\"]", + "got_raw": "dump[\"seed\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "dump[\"hd\"]", + "got": "dump[\"hd\"]", + "got_raw": "dump[\"hd\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "dump[\"ecc\"]", + "got": "dump[\"ecc\"]", + "got_raw": "dump[\"ecc\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "dump[\"hd\"]", + "got": "dump[\"hd\"]", + "got_raw": "dump[\"hd\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "dump[\"hd\"]", + "got": "dump[\"hd\"]", + "got_raw": "dump[\"hd\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "dump[\"hd\"]", + "got": "dump[\"hd\"]", + "got_raw": "dump[\"hd\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "dump[\"ecc\"]", + "got": "dump[\"ecc\"]", + "got_raw": "dump[\"ecc\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hdwallet-io/python-hdwallet", + "expected": "dump", + "got": "dump", + "got_raw": "dump", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "Min(-7))", + "got": "Min(-7))", + "got_raw": "Min(-7))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "gcd(18, 3))", + "got": "3)", + "got_raw": "3)", + "exact_match": false, + "code_bleu": 0.20203050891044216, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "Max(-7))", + "got": "Max(-7))", + "got_raw": "Max(-7))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "Max(-2, -2))", + "got": "Max(-2, -2))", + "got_raw": "Max(-2, -2))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "Max(4, -9))", + "got": "Max(4, -9))", + "got_raw": "Max(4, -9))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "4", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "Mod(9, -213))", + "got": "Mod(9, -213))", + "got_raw": "Mod(9, -213))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "Symbol('x_{i,j}' + hashlib.md5('x_{i,j}'.encode()).hexdigest(), real=True))", + "got": "x[i,j])", + "got_raw": "x[i,j])", + "exact_match": false, + "code_bleu": 0.059956985797992454, + "edit_similarity": 0.12195121951219512 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "Exception)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "Max(y, x))", + "got": "Max(y, x))", + "got_raw": "Max(y, x))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "Min(7.1, 9))", + "got": "Min(7.1, 9))", + "got_raw": "Min(7.1, 9))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "Min(1, 5))", + "got": "Min(1, 5))", + "got_raw": "Min(1, 5))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "Mod(-128, 4))", + "got": "Mod(-128, 4))", + "got_raw": "Mod(-128, 4))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "Max(1))", + "got": "Max(1))", + "got_raw": "Max(1))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "Min(12, 4))", + "got": "Min(12, 4))", + "got_raw": "Min(12, 4))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "gcd(32, 32))", + "got": "igcd(32))", + "got_raw": "igcd(32))", + "exact_match": false, + "code_bleu": 0.41239304942116134, + "edit_similarity": 0.7619047619047619 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "1 + Mod(1, 2))", + "got": "Mod(1 + 1, 2))", + "got_raw": "Mod(1 + 1, 2))", + "exact_match": false, + "code_bleu": 0.6985342056580097, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "hkust-nlp/simpleRL-reason", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "huggingface/lighteval", + "expected": "pipeline_params.remove_reasoning_tags)", + "got": "pipeline_params.reasoning_tags[0][0] == \"\")", + "got_raw": "pipeline_params.reasoning_tags[0][0] == \"\")", + "exact_match": false, + "code_bleu": 0.1350862565735141, + "edit_similarity": 0.7045454545454546 + }, + { + "repo": "huggingface/lighteval", + "expected": "task_metrics", + "got": "task_metrics", + "got_raw": "task_metrics", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "huggingface/lighteval", + "expected": "\" hello\")", + "got": "\" hello\")", + "got_raw": "\" hello\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "huggingface/lighteval", + "expected": "tokenizer.chat_template)", + "got": "tokenizer.chat_template)", + "got_raw": "tokenizer.chat_template)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "huggingface/lighteval", + "expected": "expected_config", + "got": "expected_config", + "got_raw": "expected_config", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "huggingface/lighteval", + "expected": "{\"mmlu|3\", \"mmlu:abstract_algebra|5\"}", + "got": "{\"mmlu:abstract_algebra\", \"mmlu:abstract_geometry\", \"mmlu:abstract_logic\", \"mmlu:abstract_number_theory\", \"mmlu:abstract_probability\", \"mmlu:abstract_set_theory\", \"mmlu:abstract_topology\", \"mmlu:abstract_vector_space\", \"mmlu:abstract_vector_space_with_basis\", \"mmlu:abstract_vector_space_with_basis_and_dimension\", \"mmlu:abstract_vector_space_with_basis_and_dimension_and_dimension\", \"mmlu:abstract_vector_space_with_basis_and_dimension_and_dimension_and_dimension\", \"m", + "got_raw": "{\"mmlu:abstract_algebra\", \"mmlu:abstract_geometry\", \"mmlu:abstract_logic\", \"mmlu:abstract_number_theory\", \"mmlu:abstract_probability\", \"mmlu:abstract_set_theory\", \"mmlu:abstract_topology\", \"mmlu:abstract_vector_space\", \"mmlu:abstract_vector_space_with_basis\", \"mmlu:abstract_vector_space_with_basis_and_dimension\", \"mmlu:abstract_vector_space_with_basis_and_dimension_and_dimension\", \"mmlu:abstract_vector_space_with_basis_and_dimension_and_dimension_and_dimension\", \"m", + "exact_match": false, + "code_bleu": 0.04970745472800838, + "edit_similarity": 0.09486166007905138 + }, + { + "repo": "huggingface/lighteval", + "expected": "\"Answer:\"", + "got": "\"Answer:\"", + "got_raw": "\"Answer:\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "huggingface/lighteval", + "expected": "\"\"", + "got": "\"Fitness:\\nHe is strong\"", + "got_raw": "\"Fitness:\\nHe is strong\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "huggingface/lighteval", + "expected": "captured_len_context)", + "got": "captured_len_context)", + "got_raw": "captured_len_context)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "huggingface/lighteval", + "expected": "\"hi\"", + "got": "\"hi\"", + "got_raw": "\"hi\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "huggingface/lighteval", + "expected": "[0]", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5 + }, + { + "repo": "huggingface/lighteval", + "expected": "details[\"test_reasoning_task|0\"][0][\"model_response\"][\"text_post_processed\"])", + "got": "details[0][\"post_processed_text\"] is None)", + "got_raw": "details[0][\"post_processed_text\"] is None)", + "exact_match": false, + "code_bleu": 0.29801867490082184, + "edit_similarity": 0.48739495798319327 + }, + { + "repo": "huggingface/lighteval", + "expected": "[\"storycloze:2016|0\"]", + "got": "[\"storycloze:2016|0\"]", + "got_raw": "[\"storycloze:2016|0\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "huggingface/lighteval", + "expected": "[7, 6])", + "got": "[1, 1])", + "got_raw": "[1, 1])", + "exact_match": false, + "code_bleu": 0.3303164318013807, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "huggingface/lighteval", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "huggingface/lighteval", + "expected": "1", + "got": "0.9", + "got_raw": "0.9", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "huggingface/lighteval", + "expected": "[0]", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5 + }, + { + "repo": "huggingface/lighteval", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hungpham2511/toppra", + "expected": "0.1", + "got": "1e-10", + "got_raw": "1e-10", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.25 + }, + { + "repo": "hungpham2511/toppra", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hungpham2511/toppra", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hungpham2511/toppra", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hungpham2511/toppra", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hungpham2511/toppra", + "expected": "50)", + "got": "t_desired)", + "got_raw": "t_desired)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "hungpham2511/toppra", + "expected": "(10, 5)", + "got": "(10, 5)", + "got_raw": "(10, 5)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hungpham2511/toppra", + "expected": "tac.ReturnCode.OK", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.0 + }, + { + "repo": "hungpham2511/toppra", + "expected": "[[6, 8], [6, 23], [6, 38]])", + "got": "[[3, 6], [10, 31.5], [16, 64]])", + "got_raw": "[[3, 6], [10, 31.5], [16, 64]])", + "exact_match": false, + "code_bleu": 0.24434229074248687, + "edit_similarity": 0.7241379310344828 + }, + { + "repo": "hungpham2511/toppra", + "expected": "b[i])", + "got": "b[i])", + "got_raw": "b[i])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hungpham2511/toppra", + "expected": "ValueError)", + "got": "toppra.interpolator.InterpolationError)", + "got_raw": "toppra.interpolator.InterpolationError)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.32 + }, + { + "repo": "hungpham2511/toppra", + "expected": "constraint.ConstraintType.CanonicalLinear", + "got": "\"velocity\"", + "got_raw": "\"velocity\"", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "hungpham2511/toppra", + "expected": "toppra.exceptions.BadInputVelocities)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.17909498863725698, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "hungpham2511/toppra", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "\"270\"", + "got": "\"270\"", + "got_raw": "\"270\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "42", + "got": "42", + "got_raw": "42", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "\"3\"", + "got": "{\"x\": 0, \"y\": 0, \"width\": 1920, \"height\": 1080, \"scale\": 1.0, \"transform\": 0}", + "got_raw": "{\"x\": 0, \"y\": 0, \"width\": 1920, \"height\": 1080, \"scale\": 1.0, \"transform\": 0}", + "exact_match": false, + "code_bleu": 0.04085892079136996, + "edit_similarity": 0.05 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "\"\"", + "got": "\"Other\"", + "got_raw": "\"Other\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "(\"30\", BOLD)", + "got": "(YELLOW, DIM)", + "got_raw": "(YELLOW, DIM)", + "exact_match": false, + "code_bleu": 0.32466791547509893, + "edit_similarity": 0.4 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "\"Flipped90\"", + "got": "\"Flipped90\"", + "got_raw": "\"Flipped90\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "full", + "got": "full", + "got_raw": "full", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "{\"Output\": {\"output\": \"DP-1\", \"action\": \"Off\"}}", + "got": "\"niri disable DP-1\"", + "got_raw": "\"niri disable DP-1\"", + "exact_match": false, + "code_bleu": 0.038461538461538464, + "edit_similarity": 0.24242424242424243 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "123", + "got": "123", + "got_raw": "123", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "\"0xABC\")", + "got": "0xABC)", + "got_raw": "0xABC)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "\"0x2\"", + "got": "\"0x2\"", + "got_raw": "\"0x2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "b\"command\")", + "got": "b'{\"command\": \"command\", \"status\": \"ok\"}\\n')", + "got_raw": "b'{\"command\": \"command\", \"status\": \"ok\"}\\n')", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.4 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "60", + "got": "60", + "got_raw": "60", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "\"DP-1\"", + "got": "\"DP-1\"", + "got_raw": "\"DP-1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "\"test-agent\"", + "got": "\"test-agent\"", + "got_raw": "\"test-agent\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "hyprland-community/pyprland", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ideoforms/isobar", + "expected": "[0, 1, 1, 0, 1, 0, 1, 1, 0, 1]", + "got": "[2, 3, \"a\", \"b\", None, 1]", + "got_raw": "[2, 3, \"a\", \"b\", None, 1]", + "exact_match": false, + "code_bleu": 0.13479108455103445, + "edit_similarity": 0.4727272727272727 + }, + { + "repo": "ideoforms/isobar", + "expected": "val", + "got": "val <= 1.0", + "got_raw": "val <= 1.0", + "exact_match": true, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "ideoforms/isobar", + "expected": "[2, 1, 0]", + "got": "[2, 1, 0]", + "got_raw": "[2, 1, 0]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ideoforms/isobar", + "expected": "[4, 6, 3, 1, 5, 3.5]", + "got": "[3, 6, 0, 3, 2, 0.5]", + "got_raw": "[3, 6, 0, 3, 2, 0.5]", + "exact_match": false, + "code_bleu": 0.28110041382951884, + "edit_similarity": 0.6 + }, + { + "repo": "ideoforms/isobar", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ideoforms/isobar", + "expected": "60", + "got": "60", + "got_raw": "60", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ideoforms/isobar", + "expected": "[0, 2, -1, None, 12]", + "got": "[0, 1, -1, None, 7]", + "got_raw": "[0, 1, -1, None, 7]", + "exact_match": false, + "code_bleu": 0.5923033072023248, + "edit_similarity": 0.8717948717948718 + }, + { + "repo": "ideoforms/isobar", + "expected": "[ [0, 'note_on', 24, 64, 0], [1, 'note_off', 24, 0], [1, 'note_on', 50, 64, 0], [2, 'note_off', 50, 0], [2, 'note_on', 28, 64, 0], [3, 'note_off', 28, 0], [3, 'note_on', 53, 64, 0], [4, 'note_off', 53, 0] ]", + "got": "[2, 4]", + "got_raw": "[2, 4]", + "exact_match": false, + "code_bleu": 0.031592209280356685, + "edit_similarity": 0.009433962264150943 + }, + { + "repo": "ideoforms/isobar", + "expected": "69)", + "got": "69)", + "got_raw": "69)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ideoforms/isobar", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ideoforms/isobar", + "expected": "[2, 4, 8]", + "got": "[2, 4, 8]", + "got_raw": "[2, 4, 8]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ideoforms/isobar", + "expected": "[2, 3, 5, 6, 8]", + "got": "[1, 2, 5, 1, 2, 5]", + "got_raw": "[1, 2, 5, 1, 2, 5]", + "exact_match": false, + "code_bleu": 0.23397625978961173, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "ideoforms/isobar", + "expected": "3", + "got": "4", + "got_raw": "4", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "ideoforms/isobar", + "expected": "[1, 0, -4, -3, None, None, 0.5]", + "got": "[-1, 0, 4, 6, 3, 2, 0, 0.5]", + "got_raw": "[-1, 0, 4, 6, 3, 2, 0, 0.5]", + "exact_match": false, + "code_bleu": 0.31872714733206725, + "edit_similarity": 0.7241379310344828 + }, + { + "repo": "ideoforms/isobar", + "expected": "[0, 5, 7]", + "got": "[iso.Key(\"C\", \"major\"), iso.Key(\"F\", \"minor\"), iso.Key(\"G\", \"major\")]", + "got_raw": "[iso.Key(\"C\", \"major\"), iso.Key(\"F\", \"minor\"), iso.Key(\"G\", \"major\")]", + "exact_match": false, + "code_bleu": 0.05442133807846856, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "ideoforms/isobar", + "expected": "[2, 2, 2]", + "got": "[1, 2, 3]", + "got_raw": "[1, 2, 3]", + "exact_match": false, + "code_bleu": 0.38260294162784475, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "ideoforms/isobar", + "expected": "[4, 5, 6, 7]", + "got": "[4, 4, 4, 4]", + "got_raw": "[4, 4, 4, 4]", + "exact_match": false, + "code_bleu": 0.30213753973567675, + "edit_similarity": 0.75 + }, + { + "repo": "ideoforms/isobar", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ideoforms/isobar", + "expected": "[0, None, 2, None, -1, None, 2]", + "got": "[0, 1, 2, 3, 2]", + "got_raw": "[0, 1, 2, 3, 2]", + "exact_match": false, + "code_bleu": 0.2852359479238859, + "edit_similarity": 0.5652173913043478 + }, + { + "repo": "ideoforms/isobar", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "inducer/loopy", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "inducer/loopy", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "inducer/loopy", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "inducer/loopy", + "expected": "np.where(np.abs(a) == max_val)[-1]", + "got": "np.argmax(np.abs(a))", + "got_raw": "np.argmax(np.abs(a))", + "exact_match": false, + "code_bleu": 0.41958219598107954, + "edit_similarity": 0.5555555555555556 + }, + { + "repo": "inducer/loopy", + "expected": "()", + "got": "()", + "got_raw": "()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "inducer/loopy", + "expected": "2.", + "got": "2.", + "got_raw": "2.", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "inducer/loopy", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "inducer/loopy", + "expected": "uncached_result", + "got": "uncached_result", + "got_raw": "uncached_result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "inducer/loopy", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "inducer/loopy", + "expected": "np.argmax(np.abs(a))", + "got": "np.argmax(a)", + "got_raw": "np.argmax(a)", + "exact_match": false, + "code_bleu": 0.4337840338729367, + "edit_similarity": 0.75 + }, + { + "repo": "inducer/loopy", + "expected": "np.arange(100, 110))", + "got": "np.arange(10) + 100)", + "got_raw": "np.arange(10) + 100)", + "exact_match": false, + "code_bleu": 0.4417918226831577, + "edit_similarity": 0.8 + }, + { + "repo": "inducer/loopy", + "expected": "lst0", + "got": "lst", + "got_raw": "lst", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "inducer/loopy", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "inducer/loopy", + "expected": "lp.diagnostic.ReductionIsNotTriangularError)", + "got": "RuntimeError)", + "got_raw": "RuntimeError)", + "exact_match": false, + "code_bleu": 0.17909498863725698, + "edit_similarity": 0.3508771929824561 + }, + { + "repo": "inducer/loopy", + "expected": "tol", + "got": "tol", + "got_raw": "tol", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ing-bank/popmon", + "expected": "np.max(get_bin_centers(split[1][hist_name])[0])", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.0330335309722978, + "edit_similarity": 0.041666666666666664 + }, + { + "repo": "ing-bank/popmon", + "expected": "5", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "ing-bank/popmon", + "expected": "np.nan)", + "got": "0)", + "got_raw": "0)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "ing-bank/popmon", + "expected": "49.5)", + "got": "49.5)", + "got_raw": "49.5)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ing-bank/popmon", + "expected": "1.5)", + "got": "1.5)", + "got_raw": "1.5)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ing-bank/popmon", + "expected": "str", + "got": "hist2.datatype", + "got_raw": "hist2.datatype", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "ing-bank/popmon", + "expected": "450", + "got": "int(0.45 * test_dataframe_pandas.shape[0])", + "got_raw": "int(0.45 * test_dataframe_pandas.shape[0])", + "exact_match": false, + "code_bleu": 0.09578464408619825, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "ing-bank/popmon", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ing-bank/popmon", + "expected": "len(features)", + "got": "4", + "got_raw": "4", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.0 + }, + { + "repo": "ing-bank/popmon", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ing-bank/popmon", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "ing-bank/popmon", + "expected": "0.5", + "got": "0.5", + "got_raw": "0.5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ing-bank/popmon", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ing-bank/popmon", + "expected": "1)", + "got": "0.0)", + "got_raw": "0.0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "ing-bank/popmon", + "expected": "\"Scaler(input_key='x', output_key='scaled_x')\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "ing-bank/popmon", + "expected": "check)", + "got": "check)", + "got_raw": "check)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ing-bank/popmon", + "expected": "splitA2", + "got": "splitA2", + "got_raw": "splitA2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ing-bank/popmon", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "innovatorved/subtitle", + "expected": "\"success\"", + "got": "\"success\"", + "got_raw": "\"success\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "innovatorved/subtitle", + "expected": "srt_content", + "got": "srt_content", + "got_raw": "srt_content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "innovatorved/subtitle", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "innovatorved/subtitle", + "expected": "\"test error\"", + "got": "\"test error\"", + "got_raw": "\"test error\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "innovatorved/subtitle", + "expected": "data", + "got": "data", + "got_raw": "data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "innovatorved/subtitle", + "expected": "\"First line\"", + "got": "\"First line\"", + "got_raw": "\"First line\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "innovatorved/subtitle", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "intility/fastapi-azure-auth", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "intility/fastapi-azure-auth", + "expected": "{'detail': 'Connection to Azure Entra ID is down. Unable to fetch provider configuration'}", + "got": "{'message': 'Hello World'}", + "got_raw": "{'message': 'Hello World'}", + "exact_match": false, + "code_bleu": 0.32466791547509893, + "edit_similarity": 0.27586206896551724 + }, + { + "repo": "intility/fastapi-azure-auth", + "expected": "{'detail': {'error': 'invalid_token', 'message': 'Unable to validate token'}}", + "got": "401", + "got_raw": "401", + "exact_match": false, + "code_bleu": 0.038461538461538464, + "edit_similarity": 0.0 + }, + { + "repo": "intility/fastapi-azure-auth", + "expected": "openapi_schema", + "got": "openapi_schema", + "got_raw": "openapi_schema", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "intility/fastapi-azure-auth", + "expected": "401", + "got": "401", + "got_raw": "401", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "intility/fastapi-azure-auth", + "expected": "(InvalidAuthWebSocket, UnauthorizedWebSocket))", + "got": "InvalidAuthWebSocket)", + "got_raw": "InvalidAuthWebSocket)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.6268656716417911 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "\"test.pdf\")", + "got": "path)", + "got_raw": "path)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.25 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "have)", + "got": "have)", + "got_raw": "have)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "invoice-x/invoice2data", + "expected": "False)", + "got": "compare_verified)", + "got_raw": "compare_verified)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.2608695652173913 + }, + { + "repo": "ipa-lab/hackingBuddyGPT", + "expected": "result[\"rate_limiting\"])", + "got": "result[\"headers\"])", + "got_raw": "result[\"headers\"])", + "exact_match": false, + "code_bleu": 0.45180100180492244, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "ipa-lab/hackingBuddyGPT", + "expected": "expected_operations)", + "got": "expected_operations)", + "got_raw": "expected_operations)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ipa-lab/hackingBuddyGPT", + "expected": "user_info)", + "got": "user_info)", + "got_raw": "user_info)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ipa-lab/hackingBuddyGPT", + "expected": "\"Test\")", + "got": "\"Test\")", + "got_raw": "\"Test\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ipa-lab/hackingBuddyGPT", + "expected": "len(prompt_history))", + "got": "len(prompt_history))", + "got_raw": "len(prompt_history))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ipa-lab/hackingBuddyGPT", + "expected": "403)", + "got": "403)", + "got_raw": "403)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ipa-lab/hackingBuddyGPT", + "expected": "expected_output_path)", + "got": "expected_output_path)", + "got_raw": "expected_output_path)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ipa-lab/hackingBuddyGPT", + "expected": "actual_prompt[0].get(\"content\"))", + "got": "actual_prompt)", + "got_raw": "actual_prompt)", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "ipa-lab/hackingBuddyGPT", + "expected": "updated_endpoints)", + "got": "updated_endpoints)", + "got_raw": "updated_endpoints)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ipa-lab/hackingBuddyGPT", + "expected": "headers)", + "got": "headers)", + "got_raw": "headers)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ipa-lab/hackingBuddyGPT", + "expected": "code)", + "got": "code)", + "got_raw": "code)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "irgolic/AutoPR", + "expected": "[ Issue( open=True, number=1, title=\"Ups an issue occurred.\", author=\"user1\", timestamp=\"2023-08-20T10:25:48Z\", messages=[ Message( body=\"I am an issue. Resolve me.\", author=\"user1\", ) ], ) ]", + "got": "[Issue(1, \"title\", \"body1\", \"user1\", \"2023-08-19T17:38:34Z\", \"2023-08-20T10:25:48Z\", 0, comments_url)]", + "got_raw": "[Issue(1, \"title\", \"body1\", \"user1\", \"2023-08-19T17:38:34Z\", \"2023-08-20T10:25:48Z\", 0, comments_url)]", + "exact_match": false, + "code_bleu": 0.0969369373544573, + "edit_similarity": 0.40273037542662116 + }, + { + "repo": "irgolic/AutoPR", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "irgolic/AutoPR", + "expected": "\"My commit message\\n\"", + "got": "\"test commit\"", + "got_raw": "\"test commit\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5294117647058824 + }, + { + "repo": "irgolic/AutoPR", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "isso-comments/isso", + "expected": "403)", + "got": "429)", + "got_raw": "429)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "isso-comments/isso", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "isso-comments/isso", + "expected": "142)", + "got": "142)", + "got_raw": "142)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "isso-comments/isso", + "expected": "convert.render(_in))", + "got": "convert(_in))", + "got_raw": "convert(_in))", + "exact_match": false, + "code_bleu": 0.5680505205478933, + "edit_similarity": 0.7878787878787878 + }, + { + "repo": "isso-comments/isso", + "expected": "a[\"id\"])", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.2149139863647084, + "edit_similarity": 0.2 + }, + { + "repo": "isso-comments/isso", + "expected": "[(None,), (\"foo\",)])", + "got": "[(\"\",), (\"foo\",)])", + "got_raw": "[(\"\",), (\"foo\",)])", + "exact_match": false, + "code_bleu": 0.7910665071754358, + "edit_similarity": 0.8421052631578947 + }, + { + "repo": "isso-comments/isso", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "isso-comments/isso", + "expected": "\"Foo, Bar\")", + "got": "\"Content-Type, X-Isso-Id, X-Isso-Password, X-Isso-User, X-Isso-Name, X-Isso-Email, X-Isso-Url, X-Isso-Title, X-Isso-Description, X-Isso-Icon, X-Isso-Theme, X-Isso-Theme-Path, X-Isso-Theme-Color, X-Isso-Theme-Font, X-Isso-Theme-Font-Size, X-Isso-Theme-Font-Weight, X-Isso-", + "got_raw": "\"Content-Type, X-Isso-Id, X-Isso-Password, X-Isso-User, X-Isso-Name, X-Isso-Email, X-Isso-Url, X-Isso-Title, X-Isso-Description, X-Isso-Icon, X-Isso-Theme, X-Isso-Theme-Path, X-Isso-Theme-Color, X-Isso-Theme-Font, X-Isso-Theme-Font-Size, X-Isso-Theme-Font-Weight, X-Isso-", + "exact_match": false, + "code_bleu": 0.00888932789275958, + "edit_similarity": 0.0425531914893617 + }, + { + "repo": "isso-comments/isso", + "expected": "WordPress)", + "got": "WordPress)", + "got_raw": "WordPress)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "isso-comments/isso", + "expected": "[\"fgh\"])", + "got": "[\"fgh\"])", + "got_raw": "[\"fgh\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jackmpcollins/magentic", + "expected": "\"Ireland\"", + "got": "\"Ireland\"", + "got_raw": "\"Ireland\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jackmpcollins/magentic", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jackmpcollins/magentic", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jackmpcollins/magentic", + "expected": "ValueError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "jackmpcollins/magentic", + "expected": "{\"my-extra-header\": \"foo\"}", + "got": "{\"my-extra-header\": \"foo\"}", + "got_raw": "{\"my-extra-header\": \"foo\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jackmpcollins/magentic", + "expected": "[ UserMessage(\"Hello World.\") ]", + "got": "[UserMessage(\"Hello World.\").stop()]", + "got_raw": "[UserMessage(\"Hello World.\").stop()]", + "exact_match": false, + "code_bleu": 0.5022573937283119, + "edit_similarity": 0.8656716417910447 + }, + { + "repo": "jackmpcollins/magentic", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "jackmpcollins/magentic", + "expected": "list(expected)", + "got": "expected", + "got_raw": "expected", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "jackmpcollins/magentic", + "expected": "int)", + "got": "str)", + "got_raw": "str)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "jackmpcollins/magentic", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "jackmpcollins/magentic", + "expected": "UserImageMessage[str])", + "got": "UserImageMessage)", + "got_raw": "UserImageMessage)", + "exact_match": false, + "code_bleu": 0.23784142300054423, + "edit_similarity": 0.8717948717948718 + }, + { + "repo": "jackmpcollins/magentic", + "expected": "NumberedList[str])", + "got": "NumberedList)", + "got_raw": "NumberedList)", + "exact_match": false, + "code_bleu": 0.23784142300054423, + "edit_similarity": 0.8387096774193549 + }, + { + "repo": "jackmpcollins/magentic", + "expected": "\"country\"", + "got": "\"country\"", + "got_raw": "\"country\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jackmpcollins/magentic", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "\"nb-123\"", + "got": "\"nb-123\"", + "got_raw": "\"nb-123\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "\"art-3\"", + "got": "\"art-3\"", + "got_raw": "\"art-3\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "\"CYK0Xb\"", + "got": "'create_note'", + "got_raw": "'create_note'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "\"CCqFvf\"", + "got": "\"create_notebook\"", + "got_raw": "\"create_notebook\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.16 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "constants.FLASHCARD_DIFFICULTIES.names", + "got": "constants.FLASHCARDS_VERB_DEFAULTS", + "got_raw": "constants.FLASHCARDS_VERB_DEFAULTS", + "exact_match": false, + "code_bleu": 0.3567621345008163, + "edit_similarity": 0.7222222222222222 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "\"123456789\"", + "got": "\"123456789\"", + "got_raw": "\"123456789\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "str(err)", + "got": "str(err)", + "got_raw": "str(err)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "body", + "got": "body", + "got_raw": "body", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "\"art-8\"", + "got": "\"art-8\"", + "got_raw": "\"art-8\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "\"abc-123\"", + "got": "\"abc-123\"", + "got_raw": "\"abc-123\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "constants.VIDEO_STYLES.names", + "got": "constants.VIDEO_VERB_DEFAULTS[\"style\"]", + "got_raw": "constants.VIDEO_VERB_DEFAULTS[\"style\"]", + "exact_match": false, + "code_bleu": 0.2907153684841096, + "edit_similarity": 0.5757575757575758 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "\"sheets\"", + "got": "\"sheets\"", + "got_raw": "\"sheets\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "\"task-123\"", + "got": "\"task-123\"", + "got_raw": "\"task-123\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "\"video\"", + "got": "\"video\"", + "got_raw": "\"video\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "\"public\"", + "got": "\"public\"", + "got_raw": "\"public\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "\"url\"", + "got": "\"url\"", + "got_raw": "\"url\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "\"completed\"", + "got": "\"completed\"", + "got_raw": "\"completed\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jacob-bd/notebooklm-mcp-cli", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jamwithai/production-agentic-rag-course", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jamwithai/production-agentic-rag-course", + "expected": "\"retrieve_papers\"", + "got": "\"retrieval\"", + "got_raw": "\"retrieval\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "jamwithai/production-agentic-rag-course", + "expected": "ValueError, match=\"Query cannot be empty\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.41509433962264153 + }, + { + "repo": "jamwithai/production-agentic-rag-course", + "expected": "\"http://localhost:9200\"", + "got": "\"localhost\"", + "got_raw": "\"localhost\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6470588235294118 + }, + { + "repo": "jamwithai/production-agentic-rag-course", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jamwithai/production-agentic-rag-course", + "expected": "150", + "got": "150", + "got_raw": "150", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jamwithai/production-agentic-rag-course", + "expected": "custom_fields", + "got": "custom_fields", + "got_raw": "custom_fields", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jamwithai/production-agentic-rag-course", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jamwithai/production-agentic-rag-course", + "expected": "75", + "got": "75", + "got_raw": "75", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jamwithai/production-agentic-rag-course", + "expected": "\"2024.0001v1.pdf\"", + "got": "\"2024.0001v1.pdf\"", + "got_raw": "\"2024.0001v1.pdf\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jamwithai/production-agentic-rag-course", + "expected": "PDFValidationError)", + "got": "PDFValidationError)", + "got_raw": "PDFValidationError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jamwithai/production-agentic-rag-course", + "expected": "[\"http://localhost:11434\", \"http://ollama:11434\"]", + "got": "expected_host", + "got_raw": "expected_host", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.1935483870967742 + }, + { + "repo": "jamwithai/production-agentic-rag-course", + "expected": "0.0", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "jamwithai/production-agentic-rag-course", + "expected": "422", + "got": "422", + "got_raw": "422", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jannisborn/paperscraper", + "expected": "\"Kacper Wyrwal\"", + "got": "\"J. M. M. de Oliveira\"", + "got_raw": "\"J. M. M. de Oliveira\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.21621621621621623 + }, + { + "repo": "jannisborn/paperscraper", + "expected": "\"Juan M. Galeazzi\"", + "got": "\"J. M. M. de Oliveira\"", + "got_raw": "\"J. M. M. de Oliveira\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.45 + }, + { + "repo": "jannisborn/paperscraper", + "expected": "-1", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "jannisborn/paperscraper", + "expected": "5", + "got": "4", + "got_raw": "4", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "jannisborn/paperscraper", + "expected": "0", + "got": "0.0", + "got_raw": "0.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "jazzband/django-axes", + "expected": "3)", + "got": "2)", + "got_raw": "2)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "jazzband/django-axes", + "expected": "[warning])", + "got": "[warning])", + "got_raw": "[warning])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/django-axes", + "expected": "self.SUCCESS_RESPONSE.content)", + "got": "self.SUCCESS_RESPONSE.content)", + "got_raw": "self.SUCCESS_RESPONSE.content)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/django-axes", + "expected": "threshold_none)", + "got": "42)", + "got_raw": "42)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.1111111111111111 + }, + { + "repo": "jazzband/django-axes", + "expected": "0)", + "got": "1, AxesProxyHandler.reset_attempts(username=self.USERNAME_2))", + "got_raw": "1, AxesProxyHandler.reset_attempts(username=self.USERNAME_2))", + "exact_match": false, + "code_bleu": 0.09552040806823771, + "edit_similarity": 0.031746031746031744 + }, + { + "repo": "jazzband/django-axes", + "expected": "5)", + "got": "0)", + "got_raw": "0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "jazzband/django-rest-knox", + "expected": "32", + "got": "32", + "got_raw": "32", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/django-rest-knox", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/django-rest-knox", + "expected": "'Token'", + "got": "'Bearer'", + "got_raw": "'Bearer'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "jazzband/django-rest-knox", + "expected": "bytes)", + "got": "bytes)", + "got_raw": "bytes)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/django-silk", + "expected": "200)", + "got": "200)", + "got_raw": "200)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/django-silk", + "expected": "params)", + "got": "self._query().sql)", + "got_raw": "self._query().sql)", + "exact_match": false, + "code_bleu": 0.16036590969929357, + "edit_similarity": 0.16 + }, + { + "repo": "jazzband/django-silk", + "expected": "p.name == name)", + "got": "name in p.name)", + "got_raw": "name in p.name)", + "exact_match": false, + "code_bleu": 0.41907687637277224, + "edit_similarity": 0.4666666666666667 + }, + { + "repo": "jazzband/django-silk", + "expected": "round(c))", + "got": "c)", + "got_raw": "c)", + "exact_match": false, + "code_bleu": 0.28284271247461906, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "jazzband/django-silk", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/django-silk", + "expected": "context)", + "got": "context)", + "got_raw": "context)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/django-silk", + "expected": "len(results))", + "got": "len(results))", + "got_raw": "len(results))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/django-silk", + "expected": "len(DataCollector().queries))", + "got": "self.collector.count)", + "got_raw": "self.collector.count)", + "exact_match": false, + "code_bleu": 0.16297434081163364, + "edit_similarity": 0.48 + }, + { + "repo": "jazzband/django-silk", + "expected": "expected_regex)", + "got": "expected_regex)", + "got_raw": "expected_regex)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/django-silk", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/django-silk", + "expected": "mock_DataCollector.return_value.register_profile.call_count)", + "got": "mock_DataCollector.called)", + "got_raw": "mock_DataCollector.called)", + "exact_match": false, + "code_bleu": 0.25406637407730737, + "edit_similarity": 0.5581395348837209 + }, + { + "repo": "jazzband/django-silk", + "expected": "code == 200)", + "got": "code == 200)", + "got_raw": "code == 200)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/django-silk", + "expected": "13.8)", + "got": "3.3)", + "got_raw": "3.3)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "jazzband/django-silk", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/django-silk", + "expected": "110)", + "got": "200)", + "got_raw": "200)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "jazzband/django-silk", + "expected": "'normal_uri_')", + "got": "'normal')", + "got_raw": "'normal')", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.782608695652174 + }, + { + "repo": "jazzband/djangorestframework-simplejwt", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/djangorestframework-simplejwt", + "expected": "946684800)", + "got": "datetime_to_epoch(datetime(year=2020, month=1, day=1)))", + "got_raw": "datetime_to_epoch(datetime(year=2020, month=1, day=1)))", + "exact_match": false, + "code_bleu": 0.06809398432036522, + "edit_similarity": 0.09230769230769231 + }, + { + "repo": "jazzband/djangorestframework-simplejwt", + "expected": "\"TokenUser 42\")", + "got": "\"deep-thought\")", + "got_raw": "\"deep-thought\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "jazzband/djangorestframework-simplejwt", + "expected": "TokenError)", + "got": "TokenError)", + "got_raw": "TokenError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/djangorestframework-simplejwt", + "expected": "res.data[\"code\"])", + "got": "res.data[\"detail\"])", + "got_raw": "res.data[\"detail\"])", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "jazzband/djangorestframework-simplejwt", + "expected": "hash(self.user.id))", + "got": "hash(self.token))", + "got_raw": "hash(self.token))", + "exact_match": false, + "code_bleu": 0.4624694336121693, + "edit_similarity": 0.7222222222222222 + }, + { + "repo": "jazzband/djangorestframework-simplejwt", + "expected": "self.payload)", + "got": "self.payload)", + "got_raw": "self.payload)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/djangorestframework-simplejwt", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jazzband/djangorestframework-simplejwt", + "expected": "token.payload)", + "got": "token)", + "got_raw": "token)", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.6 + }, + { + "repo": "jdemaeyer/brightsky", + "expected": "[ { 'url': k, 'parser': v[0], 'last_modified': datetime.datetime.strptime( v[1], '%Y-%m-%d %H:%M').replace(tzinfo=tzutc()), 'file_size': v[2], } for k, v in expected.items()]", + "got": "expected", + "got_raw": "expected", + "exact_match": false, + "code_bleu": 0.010431641359672991, + "edit_similarity": 0.08791208791208792 + }, + { + "repo": "jdemaeyer/brightsky", + "expected": "237667", + "got": "1200 * 1100", + "got_raw": "1200 * 1100", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "jdemaeyer/brightsky", + "expected": "[ [13, 11, 8, 9, 9], [12, 11, 7, 8, 9], [9, 10, 8, 8, 9], [0, 0, 2, 6, 8], [0, 0, 0, 0, 0], ]", + "got": "[[-1, -1, -1, -1, -1], [-1, -1, -1, -1, -1], [-1, -1, -1, -1, -1], [-1, -1, -1, -1, -1], [-1, -1, -1, -1, -1]]", + "got_raw": "[[-1, -1, -1, -1, -1], [-1, -1, -1, -1, -1], [-1, -1, -1, -1, -1], [-1, -1, -1, -1, -1], [-1, -1, -1, -1, -1]]", + "exact_match": false, + "code_bleu": 0.0767121143844601, + "edit_similarity": 0.5812807881773399 + }, + { + "repo": "jdemaeyer/brightsky", + "expected": "db_sources[0]['id'] + 2", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.0660670619445956, + "edit_similarity": 0.08333333333333333 + }, + { + "repo": "jdemaeyer/brightsky", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jdemaeyer/brightsky", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "jdepoix/youtube-transcript-api", + "expected": "CookieInvalid)", + "got": "CookiePathInvalid)", + "got_raw": "CookiePathInvalid)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.875 + }, + { + "repo": "jdepoix/youtube-transcript-api", + "expected": "\"\")", + "got": "self.transcript_raw[-1][\"text\"])", + "got_raw": "self.transcript_raw[-1][\"text\"])", + "exact_match": false, + "code_bleu": 0.11390778025531027, + "edit_similarity": 0.17142857142857143 + }, + { + "repo": "jdepoix/youtube-transcript-api", + "expected": "IpBlocked)", + "got": "YouTubeRequestFailed)", + "got_raw": "YouTubeRequestFailed)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.25806451612903225 + }, + { + "repo": "jdepoix/youtube-transcript-api", + "expected": "NotImplementedError)", + "got": "NotImplementedError)", + "got_raw": "NotImplementedError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jdepoix/youtube-transcript-api", + "expected": "\"ar\")", + "got": "\"ar\")", + "got_raw": "\"ar\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeertmans/manim-slides", + "expected": "api_name", + "got": "api_name", + "got_raw": "api_name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeertmans/manim-slides", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeertmans/manim-slides", + "expected": "results.stdout", + "got": "results.output", + "got_raw": "results.output", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7857142857142857 + }, + { + "repo": "jeertmans/manim-slides", + "expected": "result.output", + "got": "result.output", + "got_raw": "result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jeertmans/manim-slides", + "expected": "ImportError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5217391304347826 + }, + { + "repo": "jeertmans/manim-slides", + "expected": "3", + "got": "4", + "got_raw": "4", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "joke2k/django-environ", + "expected": "engine", + "got": "engine", + "got_raw": "engine", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/django-environ", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/django-environ", + "expected": "str(excinfo.value)", + "got": "str(excinfo.value)", + "got_raw": "str(excinfo.value)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/django-environ", + "expected": "backend", + "got": "backend", + "got_raw": "backend", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/django-environ", + "expected": "587", + "got": "587", + "got_raw": "587", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/django-environ", + "expected": "1234", + "got": "1234", + "got_raw": "1234", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/django-environ", + "expected": "{'TIMEOUT': 30}", + "got": "{'timeout': 30}", + "got_raw": "{'timeout': 30}", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "joke2k/django-environ", + "expected": "\"channels_redis.core.RedisChannelLayer\"", + "got": "\"channels.layers.RedisChannelLayer\"", + "got_raw": "\"channels.layers.RedisChannelLayer\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8108108108108109 + }, + { + "repo": "joke2k/django-environ", + "expected": "None", + "got": "Env.ENVIRON", + "got_raw": "Env.ENVIRON", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "joke2k/django-environ", + "expected": "old", + "got": "old", + "got_raw": "old", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/faker", + "expected": "\"0\"", + "got": "\"0\"", + "got_raw": "\"0\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/faker", + "expected": "states", + "got": "states", + "got_raw": "states", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/faker", + "expected": "(0,)", + "got": "(0,)", + "got_raw": "(0,)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/faker", + "expected": "[DEFAULT_LOCALE]", + "got": "faker_locale()", + "got_raw": "faker_locale()", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.06666666666666667 + }, + { + "repo": "joke2k/faker", + "expected": "{\"A\", \"B\", \"C\"}", + "got": "elements", + "got_raw": "elements", + "exact_match": false, + "code_bleu": 0.07142857142857142, + "edit_similarity": 0.0 + }, + { + "repo": "joke2k/faker", + "expected": "self.cryptocurrencies", + "got": "self.cryptocurrencies", + "got_raw": "self.cryptocurrencies", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/faker", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/faker", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/faker", + "expected": "UaProvider.apple_devices", + "got": "UaProvider.apple_devices", + "got_raw": "UaProvider.apple_devices", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/faker", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "joke2k/faker", + "expected": "{\"A\", \"B\"}", + "got": "elements", + "got_raw": "elements", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.0 + }, + { + "repo": "joke2k/faker", + "expected": "\"xml\"", + "got": "\"xmltodict\"", + "got_raw": "\"xmltodict\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.625 + }, + { + "repo": "joke2k/faker", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/faker", + "expected": "\"X\"", + "got": "\"9\"", + "got_raw": "\"9\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "joke2k/faker", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/faker", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/faker", + "expected": "\"3\"", + "got": "\"0\"", + "got_raw": "\"0\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "joke2k/faker", + "expected": "random", + "got": "random", + "got_raw": "random", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/faker", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joke2k/faker", + "expected": "count", + "got": "count", + "got_raw": "count", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jonatasgrosman/findpapers", + "expected": "paper_entry.get(\"issn\")", + "got": "paper_entry.get(\"issn\")", + "got_raw": "paper_entry.get(\"issn\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jonatasgrosman/findpapers", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jonatasgrosman/findpapers", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jonatasgrosman/findpapers", + "expected": "paper_entry.get(\"publication_title\")", + "got": "\"Title fake\"", + "got_raw": "\"Title fake\"", + "exact_match": false, + "code_bleu": 0.08333333333333333, + "edit_similarity": 0.25 + }, + { + "repo": "jonatasgrosman/findpapers", + "expected": "numeric_format", + "got": "numeric_format", + "got_raw": "numeric_format", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jonatasgrosman/findpapers", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jonatasgrosman/findpapers", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jonghwanhyeon/python-ffmpeg", + "expected": "target[\"streams\"][0][\"codec_name\"]", + "got": "target[\"streams\"][0][\"codec_name\"]", + "got_raw": "target[\"streams\"][0][\"codec_name\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jonghwanhyeon/python-ffmpeg", + "expected": "[\"-y\", \"-codec\", \"copy\", \"-i\", os.fspath(pier39_path)]", + "got": "[\"-y\"]", + "got_raw": "[\"-y\"]", + "exact_match": false, + "code_bleu": 0.11980183204617609, + "edit_similarity": 0.2 + }, + { + "repo": "jonghwanhyeon/python-ffmpeg", + "expected": "target[\"streams\"][0][\"codec_name\"]", + "got": "target[\"streams\"][0][\"codec_name\"]", + "got_raw": "target[\"streams\"][0][\"codec_name\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jonghwanhyeon/python-ffmpeg", + "expected": "target[\"format\"][\"format_name\"]", + "got": "target[\"format\"][\"format_name\"]", + "got_raw": "target[\"format\"][\"format_name\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jonghwanhyeon/python-ffmpeg", + "expected": "FFmpegUnsupportedCodec)", + "got": "FFmpegUnsupportedCodec)", + "got_raw": "FFmpegUnsupportedCodec)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joouha/euporie", + "expected": "invalidate_count", + "got": "invalidate_count", + "got_raw": "invalidate_count", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joouha/euporie", + "expected": "sized_datum", + "got": "size", + "got_raw": "size", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "joouha/euporie", + "expected": "(10, 20)", + "got": "(10, 20)", + "got_raw": "(10, 20)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joouha/euporie", + "expected": "\"a\"", + "got": "\" \"", + "got_raw": "\" \"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "joouha/euporie", + "expected": "\"", + "got": "\"", + "got_raw": "\"#123456\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joouha/euporie", + "expected": "[]", + "got": "(10,)", + "got_raw": "(10,)", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.0 + }, + { + "repo": "joouha/euporie", + "expected": "0.25", + "got": "16 / 8", + "got_raw": "16 / 8", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "joouha/euporie", + "expected": "\"c\"", + "got": "\"c\"", + "got_raw": "\"c\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joouha/euporie", + "expected": "\"text/html\"", + "got": "\"text/html\"", + "got_raw": "\"text/html\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joouha/euporie", + "expected": "[ (\"ansigreen\", \"H\"), (\"ansigreen\", \"e\"), (\"ansigreen bg:ansimagenta\", \"l\"), (\"ansigreen bg:ansimagenta\", \"l\"), (\"ansigreen bg:ansimagenta\", \"o\"), ]", + "got": "[(\"class:ansi-32\", \"He\"), (\"class:ansi-45\", \"ll\"), (\"class:ansi-32\", \"o\")]", + "got_raw": "[(\"class:ansi-32\", \"He\"), (\"class:ansi-45\", \"ll\"), (\"class:ansi-32\", \"o\")]", + "exact_match": false, + "code_bleu": 0.15050040762486588, + "edit_similarity": 0.46846846846846846 + }, + { + "repo": "joouha/euporie", + "expected": "\"my_cmd_2\"", + "got": "\"my_cmd_2\"", + "got_raw": "\"my_cmd_2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joouha/euporie", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joouha/euporie", + "expected": "\"value\"", + "got": "\"value\"", + "got_raw": "\"value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "joouha/euporie", + "expected": "{x[0] for x in result}", + "got": "result[1].style", + "got_raw": "result[1].style", + "exact_match": false, + "code_bleu": 0.15761459366544947, + "edit_similarity": 0.32432432432432434 + }, + { + "repo": "joouha/euporie", + "expected": "async_utils._LOOPS", + "got": "async_utils._LOOPS", + "got_raw": "async_utils._LOOPS", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jorisroovers/gitlint", + "expected": "1)", + "got": "2)", + "got_raw": "2)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "jorisroovers/gitlint", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jorisroovers/gitlint", + "expected": "3)", + "got": "0)", + "got_raw": "0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "jorisroovers/gitlint", + "expected": "[\"T1\", \"B3\"])", + "got": "[\"T1\", \"B3\"])", + "got_raw": "[\"T1\", \"B3\"])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jorisroovers/gitlint", + "expected": "config)", + "got": "config)", + "got_raw": "config)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jorisroovers/gitlint", + "expected": "OrderedDict({\"my-r\u00fcle\": expected}))", + "got": "{expected.id: expected})", + "got_raw": "{expected.id: expected})", + "exact_match": false, + "code_bleu": 0.41523375798062223, + "edit_similarity": 0.576271186440678 + }, + { + "repo": "jorisroovers/gitlint", + "expected": "\"core.commentchar\")", + "got": "\"core.commentchar\")", + "got_raw": "\"core.commentchar\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jorisroovers/gitlint", + "expected": "violations)", + "got": "violations)", + "got_raw": "violations)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jorisroovers/gitlint", + "expected": "None)", + "got": "\"commit\")", + "got_raw": "\"commit\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "jorisroovers/gitlint", + "expected": "user_rules_path)", + "got": "user_rules_path)", + "got_raw": "user_rules_path)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jorisroovers/gitlint", + "expected": "expected_log_messages)", + "got": "expected_log_messages)", + "got_raw": "expected_log_messages)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jorisroovers/gitlint", + "expected": "72)", + "got": "72)", + "got_raw": "72)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jorisroovers/gitlint", + "expected": "rules.BodyHardTab())", + "got": "rules.BodyHardTab())", + "got_raw": "rules.BodyHardTab())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jorisroovers/gitlint", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jorisroovers/gitlint", + "expected": "120)", + "got": "120)", + "got_raw": "120)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jorisroovers/gitlint", + "expected": "\"f\u00f6o\")", + "got": "\"f\u00f6o\")", + "got_raw": "\"f\u00f6o\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jorisroovers/gitlint", + "expected": "len(class_names))", + "got": "len(class_names))", + "got_raw": "len(class_names))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jorisroovers/gitlint", + "expected": "stdout.getvalue())", + "got": "stdout.getvalue())", + "got_raw": "stdout.getvalue())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "303", + "got": "303", + "got_raw": "303", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "303", + "got": "303", + "got_raw": "303", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "303", + "got": "303", + "got_raw": "303", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "tuple(expected_output)", + "got": "expected_output", + "got_raw": "expected_output", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.8108108108108109 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "\"John Doe\"", + "got": "\"John Doe\"", + "got_raw": "\"John Doe\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "\"Edited Record\"", + "got": "\"Edited Record\"", + "got_raw": "\"Edited Record\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "ValueError, match=re.escape( \"Invalid argument, Expected Tuple[str | InstrumentedAttribute, bool]\" ))", + "got": "InvalidModelError, match=\"Invalid default sort list\")", + "got_raw": "InvalidModelError, match=\"Invalid default sort list\")", + "exact_match": false, + "code_bleu": 0.23712047560635896, + "edit_similarity": 0.4025974025974026 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "\"Other label\"", + "got": "\"Other label\"", + "got_raw": "\"Other label\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "303", + "got": "303", + "got_raw": "303", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "[ \"Do some magic\", \"Do something nice\", ]", + "got": "[\"Do some magic\", \"Do something nice\"]", + "got_raw": "[\"Do some magic\", \"Do something nice\"]", + "exact_match": false, + "code_bleu": 0.6627256073058756, + "edit_similarity": 0.9620253164556962 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "303", + "got": "303", + "got_raw": "303", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "\"other-id\"", + "got": "\"other-id\"", + "got_raw": "\"other-id\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "\"other-id\"", + "got": "\"other-id\"", + "got_raw": "\"other-id\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jowilf/starlette-admin", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "jpadilla/pyjwt", + "expected": "\"nbf\"", + "got": "\"nbf\"", + "got_raw": "\"nbf\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jpadilla/pyjwt", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jpadilla/pyjwt", + "expected": "\"iss\"", + "got": "\"iss\"", + "got_raw": "\"iss\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jpadilla/pyjwt", + "expected": "\"sig\"", + "got": "\"sig\"", + "got_raw": "\"sig\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jpadilla/pyjwt", + "expected": "decoded", + "got": "decoded", + "got_raw": "decoded", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jpadilla/pyjwt", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jpsca/jinjax", + "expected": "\"Test\"", + "got": "\"Test\"", + "got_raw": "\"Test\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jpsca/jinjax", + "expected": "{\"ipsum\": 4}", + "got": "[\"ipsum\"]", + "got_raw": "[\"ipsum\"]", + "exact_match": false, + "code_bleu": 0.2710806010829534, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "jpsca/jinjax", + "expected": "None", + "got": "False", + "got_raw": "False", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "jpsca/jinjax", + "expected": "[\"arg\"]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.39640237166757364, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "jpsca/jinjax", + "expected": "\"z4 c3 a1 b2\"", + "got": "\"z4 c3 a1 z4 b2\"", + "got_raw": "\"z4 c3 a1 z4 b2\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.896551724137931 + }, + { + "repo": "jpsca/jinjax", + "expected": "catalog.prefixes[\"\"].searchpath", + "got": "catalog", + "got_raw": "catalog", + "exact_match": false, + "code_bleu": 0.07432544468767006, + "edit_similarity": 0.3684210526315789 + }, + { + "repo": "jrobichaud/django-structlog", + "expected": "user", + "got": "user", + "got_raw": "user", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jrobichaud/django-structlog", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jrobichaud/django-structlog", + "expected": "\"users:update\"", + "got": "\"users:update\"", + "got_raw": "\"users:update\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jrobichaud/django-structlog", + "expected": "201", + "got": "200", + "got_raw": "200", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "jrobichaud/django-structlog", + "expected": "\"/users/{username}/\".format( username=user.username )", + "got": "\"/users/\"", + "got_raw": "\"/users/\"", + "exact_match": false, + "code_bleu": 0.05, + "edit_similarity": 0.2903225806451613 + }, + { + "repo": "jrobichaud/django-structlog", + "expected": "\"This is a rest-framework structured log\"", + "got": "\"home\"", + "got_raw": "\"home\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2127659574468085 + }, + { + "repo": "jtesta/ssh-audit", + "expected": "['']", + "got": "['']", + "got_raw": "['']", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jtesta/ssh-audit", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jtesta/ssh-audit", + "expected": "'SSH-1.99-OpenSSH_3.6.1p2'", + "got": "'OpenSSH_3.6.1p2'", + "got_raw": "'OpenSSH_3.6.1p2'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7906976744186046 + }, + { + "repo": "jtesta/ssh-audit", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jtesta/ssh-audit", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jtesta/ssh-audit", + "expected": "p[0]", + "got": "p[0]", + "got_raw": "p[0]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jtesta/ssh-audit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jtesta/ssh-audit", + "expected": "(2, 0)", + "got": "'2.0'", + "got_raw": "'2.0'", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "jtesta/ssh-audit", + "expected": "rv", + "got": "rv", + "got_raw": "rv", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jtesta/ssh-audit", + "expected": "level", + "got": "level", + "got_raw": "level", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jtesta/ssh-audit", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "jtesta/ssh-audit", + "expected": "('libssh', '0.7.4', False)", + "got": "('libssh', '10.7.4', False)", + "got_raw": "('libssh', '10.7.4', False)", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9811320754716981 + }, + { + "repo": "karlicoss/HPI", + "expected": "pd.DataFrame(data={ 'value': ['aba' , np.nan , 'cde' , np.nan ], 'error': [np.nan, 'RuntimeError: whoops\\n', np.nan, \"RuntimeError: ('exception with datetime', datetime.datetime(2024, 8, 19, 22, 47, 1, tzinfo=datetime.timezone.utc))\\n\"],", + "got": "pd.DataFrame(data={'value': ['aba', 'cde'], 'error': ['RuntimeError: whoops\\n', 'RuntimeError: exception with datetime\\n'], 'dt': [np.nan, datetime.fromisoformat('2024-08-19T22:47:01Z')]}))", + "got_raw": "pd.DataFrame(data={'value': ['aba', 'cde'], 'error': ['RuntimeError: whoops\\n', 'RuntimeError: exception with datetime\\n'], 'dt': [np.nan, datetime.fromisoformat('2024-08-19T22:47:01Z')]}))", + "exact_match": false, + "code_bleu": 0.5171772290601552, + "edit_similarity": 0.6103286384976526 + }, + { + "repo": "karlicoss/HPI", + "expected": "timedelta(days=7.0 + 5.0, hours=5.0, minutes=10.0, seconds=50.0)", + "got": "timedelta(weeks=1, days=5, hours=5, minutes=10, seconds=50)", + "got_raw": "timedelta(weeks=1, days=5, hours=5, minutes=10, seconds=50)", + "exact_match": false, + "code_bleu": 0.2245166162074563, + "edit_similarity": 0.8130081300813008 + }, + { + "repo": "karlicoss/HPI", + "expected": "123", + "got": "val", + "got_raw": "val", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "karlicoss/HPI", + "expected": "RuntimeError)", + "got": "RuntimeError)", + "got_raw": "RuntimeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "karlicoss/HPI", + "expected": "[\"0\", \"1\", \"2\"]", + "got": "[\"0\", \"1\", \"2\"]", + "got_raw": "[\"0\", \"1\", \"2\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "karlicoss/HPI", + "expected": "[obj]", + "got": "[obj]", + "got_raw": "[obj]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "karlicoss/HPI", + "expected": "'Europe/Rome'", + "got": "'Europe/Berlin'", + "got_raw": "'Europe/Berlin'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7142857142857143 + }, + { + "repo": "karlicoss/HPI", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "karlicoss/HPI", + "expected": "{f'total_{run_id}': 1 + 123}", + "got": "item2.raw", + "got_raw": "item2.raw", + "exact_match": false, + "code_bleu": 0.1424691910596736, + "edit_similarity": 0.10810810810810811 + }, + { + "repo": "karlicoss/HPI", + "expected": "cc", + "got": "cc", + "got_raw": "cc", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "karlicoss/HPI", + "expected": "Counter({\"_A\": 4, \"Unsortable\": 3})", + "got": "Counter({\"Unsortable\": 3, \"A\": 2, \"B\": 1})", + "got_raw": "Counter({\"Unsortable\": 3, \"A\": 2, \"B\": 1})", + "exact_match": false, + "code_bleu": 0.24264382743890414, + "edit_similarity": 0.6753246753246753 + }, + { + "repo": "karlicoss/HPI", + "expected": "sqlite3.OperationalError, match='readonly database')", + "got": "sqlite3.OperationalError)", + "got_raw": "sqlite3.OperationalError)", + "exact_match": false, + "code_bleu": 0.35355339059327373, + "edit_similarity": 0.6493506493506493 + }, + { + "repo": "karlicoss/HPI", + "expected": "[123]", + "got": "[123]", + "got_raw": "[123]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "karlicoss/HPI", + "expected": "QueryException, match=r\"Cannot specify 'after', 'before' and 'within'\")", + "got": "QueryException)", + "got_raw": "QueryException)", + "exact_match": false, + "code_bleu": 0.16988673071467442, + "edit_similarity": 0.3488372093023256 + }, + { + "repo": "karlicoss/HPI", + "expected": "{ 'df': { 'count': 7, 'dtypes': { 'index': np.dtype('1\\n\"", + "got": "b\"more 2->1\\n\"", + "got_raw": "b\"more 2->1\\n\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "magic-wormhole/magic-wormhole", + "expected": "b\"hello p1\\n\"", + "got": "b\"hello p1\\n\"", + "got_raw": "b\"hello p1\\n\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "magic-wormhole/magic-wormhole", + "expected": "[addr, addr]", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.0 + }, + { + "repo": "magic-wormhole/magic-wormhole", + "expected": "wel2", + "got": "wel2", + "got_raw": "wel2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "magic-wormhole/magic-wormhole", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "magic-wormhole/magic-wormhole", + "expected": "[mock.call.start()]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.1321341238891912, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "magic-wormhole/magic-wormhole", + "expected": "TypeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "magic-wormhole/magic-wormhole", + "expected": "\"email@host\"", + "got": "\"email@host\"", + "got_raw": "\"email@host\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "magic-wormhole/magic-wormhole", + "expected": "c", + "got": "c", + "got_raw": "c", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "magic-wormhole/magic-wormhole", + "expected": "[]", + "got": "[\"phase1\"]", + "got_raw": "[\"phase1\"]", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "magic-wormhole/magic-wormhole", + "expected": "b\"hello from p2\\n\"", + "got": "b\"hello from p2\\n\"", + "got_raw": "b\"hello from p2\\n\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "magic-wormhole/magic-wormhole", + "expected": "key2", + "got": "key2", + "got_raw": "key2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "magic-wormhole/magic-wormhole", + "expected": "b\"data\"", + "got": "dataY", + "got_raw": "dataY", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "magic-wormhole/magic-wormhole", + "expected": "f\"No public keys in '{dot_ssh}'\"", + "got": "f\"Could not find a public key in {dot_ssh}\"", + "got_raw": "f\"Could not find a public key in {dot_ssh}\"", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7466666666666667 + }, + { + "repo": "magic-wormhole/magic-wormhole", + "expected": "16", + "got": "16", + "got_raw": "16", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "magic-wormhole/magic-wormhole", + "expected": "bytes", + "got": "bytes", + "got_raw": "bytes", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "magic-wormhole/magic-wormhole", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "impl_instance", + "got": "impl_instance", + "got_raw": "impl_instance", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"number\": 4}", + "got": "{\"number\": 1234567890}", + "got_raw": "{\"number\": 1234567890}", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7428571428571429 + }, + { + "repo": "maldoinc/wireup", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "42", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "help_result.output", + "got": "help_result.stdout", + "got_raw": "help_result.stdout", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "maldoinc/wireup", + "expected": "registry.impls", + "got": "registry.services", + "got_raw": "registry.services", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6451612903225806 + }, + { + "repo": "maldoinc/wireup", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"bar\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "maldoinc/wireup", + "expected": "foo.__wireup_names__", + "got": "foo.__wireup_names__", + "got_raw": "foo.__wireup_names__", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "42", + "got": "42", + "got_raw": "42", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"hello\": \"Hello World\", \"fromHeader\": \"Wireup\"}", + "got": "{\"hello\": \"Hello Wireup\"}", + "got_raw": "{\"hello\": \"Hello Wireup\"}", + "exact_match": false, + "code_bleu": 0.2984916477287617, + "edit_similarity": 0.684931506849315 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"bar_multiple_bases\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.16 + }, + { + "repo": "maldoinc/wireup", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"dependency_impl\"", + "got": "\"Hello, Dependency!\"", + "got_raw": "\"Hello, Dependency!\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5945945945945946 + }, + { + "repo": "maldoinc/wireup", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "{\"hello\": \"Hello Django\"}", + "got": "{\"hello\": \"Hello Django\"}", + "got_raw": "{\"hello\": \"Hello Django\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "5", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "scoped_service_dependency", + "got": "scoped_service_dependency", + "got_raw": "scoped_service_dependency", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "maldoinc/wireup", + "expected": "[\"after_yield\"]", + "got": "[\"before_yield\", \"after_yield\"]", + "got_raw": "[\"before_yield\", \"after_yield\"]", + "exact_match": false, + "code_bleu": 0.3860973950960897, + "edit_similarity": 0.6521739130434783 + }, + { + "repo": "maldoinc/wireup", + "expected": "\"Your lucky number is: 42\"", + "got": "\"Your lucky number is: 7\"", + "got_raw": "\"Your lucky number is: 7\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9411764705882353 + }, + { + "repo": "mammothb/symspellpy", + "expected": "si_1.term", + "got": "si_1.term", + "got_raw": "si_1.term", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mammothb/symspellpy", + "expected": "len(result)", + "got": "len(result)", + "got_raw": "len(result)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mammothb/symspellpy", + "expected": "to_similarity(6.0, length)", + "got": "to_similarity(0.5, length)", + "got_raw": "to_similarity(0.5, length)", + "exact_match": false, + "code_bleu": 0.488923022434901, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "mammothb/symspellpy", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mammothb/symspellpy", + "expected": "0.7)", + "got": "0.0)", + "got_raw": "0.0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.75 + }, + { + "repo": "mammothb/symspellpy", + "expected": "result[0].term", + "got": "[0].term", + "got_raw": "[0].term", + "exact_match": false, + "code_bleu": 0.8333333333333334, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "mammothb/symspellpy", + "expected": "TypeError)", + "got": "NotImplementedError)", + "got_raw": "NotImplementedError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "mangiucugna/json_repair", + "expected": "ValueError, match=\"doubled quotes followed by another quote while parsing a string\")", + "got": "pytest.raises(ValueError)", + "got_raw": " pytest.raises(ValueError)", + "exact_match": false, + "code_bleu": 0.24446151121745047, + "edit_similarity": 0.2018348623853211 + }, + { + "repo": "mangiucugna/json_repair", + "expected": "[1, \"x\"]", + "got": "[1, \"x\"]", + "got_raw": "[1, \"x\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mangiucugna/json_repair", + "expected": "\"[]\"", + "got": "\"[{}]\"", + "got_raw": "\"[{}]\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8 + }, + { + "repo": "mangiucugna/json_repair", + "expected": "'{\"key\": 1}'", + "got": "'{\"key\": 1.0e0}'", + "got_raw": "'{\"key\": 1.0e0}'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "mangiucugna/json_repair", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mangiucugna/json_repair", + "expected": "captured.err.lower()", + "got": "captured.err", + "got_raw": "captured.err", + "exact_match": false, + "code_bleu": 0.36038417796587763, + "edit_similarity": 0.75 + }, + { + "repo": "mangiucugna/json_repair", + "expected": "'[\"value\", \"value2\"]'", + "got": "'[\"value\", \"value2\"]'", + "got_raw": "'[\"value\", \"value2\"]'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mangiucugna/json_repair", + "expected": "\"\"", + "got": "\"{}\"", + "got_raw": "\"{}\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "mangiucugna/json_repair", + "expected": "'{\"key\": \"val\\\\n123,`key2:value2\"}'", + "got": "'{\"key\": \"val\\\\n123\", \"key2\": \"value2\"}'", + "got_raw": "'{\"key\": \"val\\\\n123\", \"key2\": \"value2\"}'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9066666666666666 + }, + { + "repo": "mangiucugna/json_repair", + "expected": "1.2", + "got": "1.2", + "got_raw": "1.2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mangiucugna/json_repair", + "expected": "'{\"key\": \"\"}'", + "got": "'{\"key\": \"\"}'", + "got_raw": "'{\"key\": \"\"}'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mangiucugna/json_repair", + "expected": "'{\"foo\": []}'", + "got": "\"{foo: []}\"", + "got_raw": "\"{foo: []}\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "mangiucugna/json_repair", + "expected": "{\"error\": \"schemaRepairMode must be a string.\"}", + "got": "{\"error\": \"Invalid schema repair mode\"}", + "got_raw": "{\"error\": \"Invalid schema repair mode\"}", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.627906976744186 + }, + { + "repo": "manjurulhoque/django-job-portal", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "manjurulhoque/django-job-portal", + "expected": "[\"This field is required.\"])", + "got": "[\"Email is required\"])", + "got_raw": "[\"Email is required\"])", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.76 + }, + { + "repo": "manjurulhoque/django-job-portal", + "expected": "\"Manjurul Hoque\")", + "got": "\"Manjurul Hoque\")", + "got_raw": "\"Manjurul Hoque\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "manjurulhoque/django-job-portal", + "expected": "User.objects.filter(email=self.valid_data[\"email\"]).exists())", + "got": "True)", + "got_raw": "True)", + "exact_match": false, + "code_bleu": 0.0537284965911771, + "edit_similarity": 0.09090909090909091 + }, + { + "repo": "manjurulhoque/django-job-portal", + "expected": "self.job.id)", + "got": "self.job.id)", + "got_raw": "self.job.id)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "manjurulhoque/django-job-portal", + "expected": "\"Invalid form\")", + "got": "\"Form is not valid\")", + "got_raw": "\"Form is not valid\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5142857142857142 + }, + { + "repo": "manjurulhoque/django-job-portal", + "expected": "\"Test Job\")", + "got": "self.job_data[\"title\"])", + "got_raw": "self.job_data[\"title\"])", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.29411764705882354 + }, + { + "repo": "manjurulhoque/django-job-portal", + "expected": "status.HTTP_400_BAD_REQUEST)", + "got": "status.HTTP_400_BAD_REQUEST)", + "got_raw": "status.HTTP_400_BAD_REQUEST)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "manzt/anywidget", + "expected": "path.read_text()", + "got": "bar", + "got_raw": "bar", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.21052631578947367 + }, + { + "repo": "manzt/anywidget", + "expected": "new_contents", + "got": "new_contents", + "got_raw": "new_contents", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "manzt/anywidget", + "expected": "bundle[0]", + "got": "bundle", + "got_raw": "bundle", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.8 + }, + { + "repo": "manzt/anywidget", + "expected": "y", + "got": "y", + "got_raw": "y", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "manzt/anywidget", + "expected": "bar", + "got": "bar", + "got_raw": "bar", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marcolivierarsenault/moonraker-home-assistant", + "expected": "1.2", + "got": "1.2", + "got_raw": "1.2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marcolivierarsenault/moonraker-home-assistant", + "expected": "caplog.text", + "got": "camera.MoonrakerCamera(hass, entry.entity_id, config_entry))", + "got_raw": "camera.MoonrakerCamera(hass, entry.entity_id, config_entry))", + "exact_match": false, + "code_bleu": 0.09552040806823771, + "edit_similarity": 0.11267605633802817 + }, + { + "repo": "marcolivierarsenault/moonraker-home-assistant", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marcolivierarsenault/moonraker-home-assistant", + "expected": "state.attributes", + "got": "state.attributes", + "got_raw": "state.attributes", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marcolivierarsenault/moonraker-home-assistant", + "expected": "0.2", + "got": "0.2", + "got_raw": "0.2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marcolivierarsenault/moonraker-home-assistant", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marcwebbie/passpie", + "expected": "DEVNULL", + "got": "DEVNULL", + "got_raw": "DEVNULL", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marcwebbie/passpie", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marcwebbie/passpie", + "expected": "credpath)", + "got": "credpath)", + "got_raw": "credpath)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marcwebbie/passpie", + "expected": "str(excinfo.value)", + "got": "str(excinfo.value)", + "got_raw": "str(excinfo.value)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marcwebbie/passpie", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marcwebbie/passpie", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marcwebbie/passpie", + "expected": "(None, \"example.com\")", + "got": "(\"\", \"example.com\")", + "got_raw": "(\"\", \"example.com\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.85 + }, + { + "repo": "marcwebbie/passpie", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marcwebbie/passpie", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marcwebbie/passpie", + "expected": "*autopull)", + "got": "autopull)", + "got_raw": "autopull)", + "exact_match": false, + "code_bleu": 0.4714045207910317, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "marcwebbie/passpie", + "expected": "5", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "marshmallow-code/apispec", + "expected": "make_schema_key(PetSchema(load_only=structure(modifier[::-1])))", + "got": "make_schema_key(PetSchema, modifier)", + "got_raw": "make_schema_key(PetSchema, modifier)", + "exact_match": false, + "code_bleu": 0.1516647134682679, + "edit_similarity": 0.6868686868686869 + }, + { + "repo": "marshmallow-code/apispec", + "expected": "resp", + "got": "{\"content\": {\"application/json\": {\"example\": {\"name\": \"Example\"}}}}", + "got_raw": "{\"content\": {\"application/json\": {\"example\": {\"name\": \"Example\"}}}}", + "exact_match": false, + "code_bleu": 0.06074588070876682, + "edit_similarity": 0.056338028169014086 + }, + { + "repo": "marshmallow-code/apispec", + "expected": "[\"field3\", \"field4\", \"field5\"]", + "got": "[\"field5\"]", + "got_raw": "[\"field5\"]", + "exact_match": false, + "code_bleu": 0.27383275896268816, + "edit_similarity": 0.5 + }, + { + "repo": "marshmallow-code/apispec", + "expected": "\"1.0.0\"", + "got": "\"1.0.0\"", + "got_raw": "\"1.0.0\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marshmallow-code/apispec", + "expected": "[ \"field5\" ]", + "got": "[\"field1\", \"field2\"]", + "got_raw": "[\"field1\", \"field2\"]", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.625 + }, + { + "repo": "marshmallow-code/apispec", + "expected": "enum", + "got": "enum", + "got_raw": "enum", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marshmallow-code/marshmallow", + "expected": "[\"name\", \"email\", \"age\", \"created\", \"id\", \"homepage\"]", + "got": "[\"id\", \"name\", \"email\"]", + "got_raw": "[\"id\", \"name\", \"email\"]", + "exact_match": false, + "code_bleu": 0.23788790452170025, + "edit_similarity": 0.4473684210526316 + }, + { + "repo": "marshmallow-code/marshmallow", + "expected": "[\"error1\", \"error2\"]", + "got": "[\"error1\", \"error2\"]", + "got_raw": "[\"error1\", \"error2\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marshmallow-code/marshmallow", + "expected": "\"invalid email\"", + "got": "\"invalid email\"", + "got_raw": "\"invalid email\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marshmallow-code/marshmallow", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marshmallow-code/marshmallow", + "expected": "{ \"_schema\": \"error2\", \"field1\": \"error1\", }", + "got": "{\"field1\": \"error1\", \"error2\": \"error2\"}", + "got_raw": "{\"field1\": \"error1\", \"error2\": \"error2\"}", + "exact_match": false, + "code_bleu": 0.43298795618809266, + "edit_similarity": 0.5238095238095238 + }, + { + "repo": "marshmallow-code/marshmallow", + "expected": "\"bar\"", + "got": "\"bar\"", + "got_raw": "\"bar\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marshmallow-code/marshmallow", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marshmallow-code/marshmallow", + "expected": "messages", + "got": "messages", + "got_raw": "messages", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marshmallow-code/marshmallow", + "expected": "{ \"f_1\": 2, \"f_2\": 1, \"f_3\": 4, \"f_4\": 1, }", + "got": "{\"f_1\": 2, \"f_2\": 3, \"f_3\": 4, \"f_4\": 5}", + "got_raw": "{\"f_1\": 2, \"f_2\": 3, \"f_3\": 4, \"f_4\": 5}", + "exact_match": false, + "code_bleu": 0.6872549579989753, + "edit_similarity": 0.9156626506024096 + }, + { + "repo": "marshmallow-code/marshmallow", + "expected": "\"baz\"", + "got": "field.attribute", + "got_raw": "field.attribute", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.1 + }, + { + "repo": "marshmallow-code/marshmallow", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marshmallow-code/webargs", + "expected": "ret", + "got": "ret", + "got_raw": "ret", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marshmallow-code/webargs", + "expected": "{\"x_field\": [{\"id\": 24}]}", + "got": "{\"X-Field\": [{\"id\": 24}]}", + "got_raw": "{\"X-Field\": [{\"id\": 24}]}", + "exact_match": false, + "code_bleu": 0.8265168183793802, + "edit_similarity": 0.88 + }, + { + "repo": "marshmallow-code/webargs", + "expected": "{}", + "got": "{\"things\": []}", + "got_raw": "{\"things\": []}", + "exact_match": false, + "code_bleu": 0.24446151121745047, + "edit_similarity": 0.25 + }, + { + "repo": "marshmallow-code/webargs", + "expected": "422", + "got": "422", + "got_raw": "422", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marshmallow-code/webargs", + "expected": "{\"name\": \"Fred\"}", + "got": "{\"name\": \"Fred\"}", + "got_raw": "{\"name\": \"Fred\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "marshmallow-code/webargs", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "var_names", + "got": "var_names", + "got_raw": "var_names", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "32", + "got": "31", + "got_raw": "31", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "(100, 8)", + "got": "[100, 20, 8]", + "got_raw": "[100, 20, 8]", + "exact_match": false, + "code_bleu": 0.29071536848410967, + "edit_similarity": 0.6 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "v2", + "got": "v2", + "got_raw": "v2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "[0, 11, 999]", + "got": "[0, 11]", + "got_raw": "[0, 11]", + "exact_match": false, + "code_bleu": 0.5680505205478933, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "data_info.n_users", + "got": "model.n_users", + "got_raw": "model.n_users", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "RuntimeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "eval_result[\"roc_auc\"]", + "got": "eval_result[\"roc_auc\"]", + "got_raw": "eval_result[\"roc_auc\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "[12, 18])", + "got": "[0, 1])", + "got_raw": "[0, 1])", + "exact_match": false, + "code_bleu": 0.3303164318013807, + "edit_similarity": 0.75 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "num_workers", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "AssertionError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5384615384615384 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "model_num", + "got": "model_num", + "got_raw": "model_num", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "(100, 3)", + "got": "(100, 3)", + "got_raw": "(100, 3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "TypeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "(6,)", + "got": "(6,)", + "got_raw": "(6,)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "massquantity/LibRecommender", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "matin/garth", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "matin/garth", + "expected": "end - timedelta(days=6)", + "got": "end", + "got_raw": "end", + "exact_match": false, + "code_bleu": 0.07432544468767006, + "edit_similarity": 0.23076923076923078 + }, + { + "repo": "matin/garth", + "expected": "1749975276000", + "got": "1749975276000", + "got_raw": "1749975276000", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "matin/garth", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "matin/garth", + "expected": "date(2021, 1, 1)", + "got": "\"2021-01-01\"", + "got_raw": "\"2021-01-01\"", + "exact_match": false, + "code_bleu": 0.0625, + "edit_similarity": 0.42857142857142855 + }, + { + "repo": "matin/garth", + "expected": "date(2026, 1, 9)", + "got": "date(2026, 1, 9)", + "got_raw": "date(2026, 1, 9)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "matin/garth", + "expected": "hr.min_heart_rate", + "got": "hr.resting_heart_rate", + "got_raw": "hr.resting_heart_rate", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.8421052631578947 + }, + { + "repo": "matin/garth", + "expected": "\"event_type\"", + "got": "\"event_type\"", + "got_raw": "\"event_type\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "matin/garth", + "expected": "name", + "got": "name", + "got_raw": "name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "matin/garth", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "matin/garth", + "expected": "expected_output", + "got": "expected_output", + "got_raw": "expected_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "matin/garth", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "matin/garth", + "expected": "ValueError, match=\"At least one of\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.46808510638297873 + }, + { + "repo": "matin/garth", + "expected": "0.99", + "got": "0.99", + "got_raw": "0.99", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "[\"a\"]", + "got": "\"1 + a\"", + "got_raw": "\"1 + a\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.5 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "out_arr)", + "got": "expected_output)", + "got_raw": "expected_output)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "2", + "got": "20", + "got_raw": "20", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "[2]", + "got": "[\"Intercept\"]", + "got_raw": "[\"Intercept\"]", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.25 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "[4]", + "got": "[2]", + "got_raw": "[2]", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "set()", + "got": "set()", + "got_raw": "set()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "[4, 5, 1]", + "got": "[1, 4]", + "got_raw": "[1, 4]", + "exact_match": false, + "code_bleu": 0.2566457942593459, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "\"0\"", + "got": "\"1 / b\"", + "got_raw": "\"1 / b\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "type(d[\"1\"])", + "got": "object", + "got_raw": "object", + "exact_match": false, + "code_bleu": 0.07142857142857142, + "edit_similarity": 0.1111111111111111 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "\"a\"", + "got": "\"ScopedFactor(Factor('a'))\"", + "got_raw": "\"ScopedFactor(Factor('a'))\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "(6, 0)", + "got": "(6, 0)", + "got_raw": "(6, 0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "d[\"1\"]", + "got": "d[\"1\"]", + "got_raw": "d[\"1\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "{\"a\"}", + "got": "[\"a\"]", + "got_raw": "[\"a\"]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.6 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "[1, 4, 5]", + "got": "[1, 4]", + "got_raw": "[1, 4]", + "exact_match": false, + "code_bleu": 0.5680505205478933, + "edit_similarity": 0.8 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "3", + "got": "20", + "got_raw": "20", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "(3, 8)", + "got": "(3, 3)", + "got_raw": "(3, 3)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "{\"a\", \"A\"}", + "got": "{\"1\", \"a\", \"A\"}", + "got_raw": "{\"1\", \"a\", \"A\"}", + "exact_match": false, + "code_bleu": 0.5410822690539396, + "edit_similarity": 0.8 + }, + { + "repo": "matthewwardrop/formulaic", + "expected": "term2", + "got": "term2", + "got_raw": "term2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mdbloice/Augmentor", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mdbloice/Augmentor", + "expected": "len(y)", + "got": "len(y)", + "got_raw": "len(y)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mdbloice/Augmentor", + "expected": "stored_ground_truth_images", + "got": "stored_ground_truth_images", + "got_raw": "stored_ground_truth_images", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mdbloice/Augmentor", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mdbloice/Augmentor", + "expected": "p_paths", + "got": "p_paths", + "got_raw": "p_paths", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mdbloice/Augmentor", + "expected": "len(glob.glob(os.path.join(tmpdir, \"*.JPEG\")))", + "got": "num_of_images", + "got_raw": "num_of_images", + "exact_match": false, + "code_bleu": 0.027777777777777776, + "edit_similarity": 0.1016949152542373 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "Exception)", + "got": "MeilisearchApiError)", + "got_raw": "MeilisearchApiError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "\"cartoon\"", + "got": "{\"genre\": \"cartoon\"}", + "got_raw": "{\"genre\": \"cartoon\"}", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.6206896551724138 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "f\"Bearer {MASTER_KEY}\"", + "got": "f\"MasterKey {MASTER_KEY}\"", + "got_raw": "f\"MasterKey {MASTER_KEY}\"", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8085106382978723 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "\"succeeded\"", + "got": "\"ok\"", + "got_raw": "\"ok\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "4096", + "got": "4096", + "got_raw": "4096", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "options[\"remotes\"][instance_name][\"url\"]", + "got": "options[\"remotes\"][instance_name][\"url\"]", + "got_raw": "options[\"remotes\"][instance_name][\"url\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "1", + "got": "5", + "got_raw": "5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "\"keyTest\"", + "got": "\"keyTest\"", + "got_raw": "\"keyTest\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "204", + "got": "200", + "got_raw": "200", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "mock_response", + "got": "mock_response", + "got_raw": "mock_response", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "\"succeeded\"", + "got": "\"succeeded\"", + "got_raw": "\"succeeded\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meilisearch/meilisearch-python", + "expected": "response", + "got": "response", + "got_raw": "response", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meta-llama/synthetic-data-kit", + "expected": "\"api-endpoint\"", + "got": "\"openai\"", + "got_raw": "\"openai\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5454545454545454 + }, + { + "repo": "meta-llama/synthetic-data-kit", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meta-llama/synthetic-data-kit", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meta-llama/synthetic-data-kit", + "expected": "context1.config", + "got": "context1.config", + "got_raw": "context1.config", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meta-llama/synthetic-data-kit", + "expected": "\"vllm\"", + "got": "\"openai\"", + "got_raw": "\"openai\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "meta-llama/synthetic-data-kit", + "expected": "output_path)", + "got": "output_path)", + "got_raw": "output_path)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meta-llama/synthetic-data-kit", + "expected": "path_obj", + "got": "str(path_obj)", + "got_raw": "str(path_obj)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.7619047619047619 + }, + { + "repo": "meta-llama/synthetic-data-kit", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meta-llama/synthetic-data-kit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meta-llama/synthetic-data-kit", + "expected": "\"test_value\"", + "got": "\"test_value\"", + "got_raw": "\"test_value\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meta-llama/synthetic-data-kit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meta-llama/synthetic-data-kit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "meta-llama/synthetic-data-kit", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "resp)", + "got": "Resp)", + "got_raw": "Resp)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8 + }, + { + "repo": "michaelhly/solana-py", + "expected": "0.0", + "got": "str(0)", + "got_raw": "str(0)", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "michaelhly/solana-py", + "expected": "0.0", + "got": "str(0)", + "got_raw": "str(0)", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "michaelhly/solana-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "michaelhly/solana-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/causica", + "expected": "torch.Size([2, 3])", + "got": "torch.Size([2, 3])", + "got_raw": "torch.Size([2, 3])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/causica", + "expected": "noise_a.mode)", + "got": "noise_a.mode.get(\"a\"))", + "got_raw": "noise_a.mode.get(\"a\"))", + "exact_match": false, + "code_bleu": 0.2777619034011791, + "edit_similarity": 0.7428571428571429 + }, + { + "repo": "microsoft/causica", + "expected": "len(metaset)", + "got": "5", + "got_raw": "5", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.0 + }, + { + "repo": "microsoft/causica", + "expected": "torch.Size([10])", + "got": "torch.Size([2])", + "got_raw": "torch.Size([2])", + "exact_match": false, + "code_bleu": 0.6606328636027614, + "edit_similarity": 0.9032258064516129 + }, + { + "repo": "microsoft/causica", + "expected": "torch.Size([2])", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.07432544468767006, + "edit_similarity": 0.125 + }, + { + "repo": "microsoft/causica", + "expected": "target_entropy)", + "got": "target_entropy)", + "got_raw": "target_entropy)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/causica", + "expected": "getattr(cf_2[0], field))", + "got": "getattr(cf_2[0], field))", + "got_raw": "getattr(cf_2[0], field))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/causica", + "expected": "np.zeros_like(logits.detach().numpy()))", + "got": "torch.zeros_like(logits))", + "got_raw": "torch.zeros_like(logits))", + "exact_match": false, + "code_bleu": 0.2774816601673016, + "edit_similarity": 0.625 + }, + { + "repo": "microsoft/causica", + "expected": "[\"x_1\"]", + "got": "torch.tensor([0.0, 0.0, 0.0, 0.0, 0.0])", + "got_raw": "torch.tensor([0.0, 0.0, 0.0, 0.0, 0.0])", + "exact_match": false, + "code_bleu": 0.08513012360883544, + "edit_similarity": 0.08695652173913043 + }, + { + "repo": "microsoft/causica", + "expected": "torch.sort(chunk[\"order\"]).values)", + "got": "slices[dim])", + "got_raw": "slices[dim])", + "exact_match": false, + "code_bleu": 0.16087391462337072, + "edit_similarity": 0.21739130434782608 + }, + { + "repo": "microsoft/causica", + "expected": "torch.tensor(0)", + "got": "0.8", + "got_raw": "0.8", + "exact_match": false, + "code_bleu": 0.08333333333333333, + "edit_similarity": 0.1111111111111111 + }, + { + "repo": "microsoft/causica", + "expected": "std(joint_log_probs))", + "got": "std(joint_log_probs))", + "got_raw": "std(joint_log_probs))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/causica", + "expected": "buffer)", + "got": "buffer)", + "got_raw": "buffer)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/causica", + "expected": "y.to(**to_kwargs))", + "got": "y.to(**to_kwargs))", + "got_raw": "y.to(**to_kwargs))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/causica", + "expected": "torch.Size([100, 8, 8])", + "got": "torch.Size([100, n, n])", + "got_raw": "torch.Size([100, n, n])", + "exact_match": false, + "code_bleu": 0.6262844962765468, + "edit_similarity": 0.9130434782608695 + }, + { + "repo": "microsoft/causica", + "expected": "torch.Size()", + "got": "self.factual_data.batch_size", + "got_raw": "self.factual_data.batch_size", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.3 + }, + { + "repo": "microsoft/causica", + "expected": "torch.tensor(-2.0))", + "got": "torch.tensor(-2.0))", + "got_raw": "torch.tensor(-2.0))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/causica", + "expected": "intervention_2[2]", + "got": "intervention_2[2]", + "got_raw": "intervention_2[2]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/causica", + "expected": "y.get(key).to(**to_kwargs))", + "got": "y.get(key))", + "got_raw": "y.get(key))", + "exact_match": false, + "code_bleu": 0.45180100180492244, + "edit_similarity": 0.5789473684210527 + }, + { + "repo": "microsoft/causica", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/mattergen", + "expected": "target_loss * 4)", + "got": "target_loss * 4)", + "got_raw": "target_loss * 4)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/mattergen", + "expected": "{\"foo\", \"bar\"}", + "got": "{\"drift\", \"diffusion\"}", + "got_raw": "{\"drift\", \"diffusion\"}", + "exact_match": false, + "code_bleu": 0.32466791547509893, + "edit_similarity": 0.5 + }, + { + "repo": "microsoft/mattergen", + "expected": "samples_fast)", + "got": "samples_fast)", + "got_raw": "samples_fast)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/mattergen", + "expected": "loss_with_bad_bar)", + "got": "loss_with_bad_bar)", + "got_raw": "loss_with_bad_bar)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/mattergen", + "expected": "t.shape", + "got": "()", + "got_raw": "()", + "exact_match": false, + "code_bleu": 0.30120066786994826, + "edit_similarity": 0.0 + }, + { + "repo": "microsoft/mattergen", + "expected": "batch[k].shape", + "got": "batch.shape", + "got_raw": "batch.shape", + "exact_match": false, + "code_bleu": 0.3194715521231362, + "edit_similarity": 0.88 + }, + { + "repo": "microsoft/mattergen", + "expected": "np.nanstd(x.numpy())", + "got": "2.0", + "got_raw": "2.0", + "exact_match": false, + "code_bleu": 0.05, + "edit_similarity": 0.08695652173913043 + }, + { + "repo": "microsoft/onnxscript", + "expected": "pattern.Pattern)", + "got": "pattern.Pattern)", + "got_raw": "pattern.Pattern)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/onnxscript", + "expected": "2)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "microsoft/onnxscript", + "expected": "5)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "microsoft/onnxscript", + "expected": "results[0])", + "got": "results[0])", + "got_raw": "results[0])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/onnxscript", + "expected": "ts.shape)", + "got": "ts.shape)", + "got_raw": "ts.shape)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/onnxscript", + "expected": "[\"Identity\"])", + "got": "[\"Slice\"])", + "got_raw": "[\"Slice\"])", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.5217391304347826 + }, + { + "repo": "microsoft/onnxscript", + "expected": "r)", + "got": "r)", + "got_raw": "r)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/onnxscript", + "expected": "model)", + "got": "ir_model)", + "got_raw": "ir_model)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8 + }, + { + "repo": "microsoft/onnxscript", + "expected": "\"\")", + "got": "\"\")", + "got_raw": "\"\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/onnxscript", + "expected": "pres)", + "got": "pres)", + "got_raw": "pres)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/onnxscript", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/onnxscript", + "expected": "np.array([8]))", + "got": "2 * M)", + "got_raw": "2 * M)", + "exact_match": false, + "code_bleu": 0.15969071642803745, + "edit_similarity": 0.1 + }, + { + "repo": "microsoft/onnxscript", + "expected": "op_types)", + "got": "op_types)", + "got_raw": "op_types)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/onnxscript", + "expected": "function.__name__)", + "got": "function.__name__)", + "got_raw": "function.__name__)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/onnxscript", + "expected": "m)", + "got": "m)", + "got_raw": "m)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/onnxscript", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/onnxscript", + "expected": "str(e))", + "got": ".fail(f\"Failed to run the model: {e}\")", + "got_raw": ".fail(f\"Failed to run the model: {e}\")", + "exact_match": false, + "code_bleu": 0.24446151121745047, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "microsoft/onnxscript", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/onnxscript", + "expected": "b)", + "got": "b)", + "got_raw": "b)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "microsoft/onnxscript", + "expected": "7)", + "got": "11)", + "got_raw": "11)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4 + }, + { + "repo": "mido/mido", + "expected": "[[0xf0, 1, 2, 3, 0xf7]]", + "got": "[0xf0, 1, 2, 3, 0xf7]", + "got_raw": "[0xf0, 1, 2, 3, 0xf7]", + "exact_match": false, + "code_bleu": 0.8461538461538461, + "edit_similarity": 0.9545454545454546 + }, + { + "repo": "mido/mido", + "expected": "(0, 1, 2)", + "got": "(0, 1, 2)", + "got_raw": "(0, 1, 2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mido/mido", + "expected": "m2", + "got": "m2", + "got_raw": "m2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mido/mido", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mido/mido", + "expected": "Message('clock', time=0)", + "got": "Message('clock', time=0)", + "got_raw": "Message('clock', time=0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mido/mido", + "expected": "[[0xf8], [0xf8]]", + "got": "[0xf8, 0xf8]", + "got_raw": "[0xf8, 0xf8]", + "exact_match": false, + "code_bleu": 0.26270600250088216, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "mido/mido", + "expected": "0", + "got": "MIN_PITCHWHEEL + 64", + "got_raw": "MIN_PITCHWHEEL + 64", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "mido/mido", + "expected": "TypeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "mido/mido", + "expected": "Message('start', time=1)", + "got": "Message('start', time=1)", + "got_raw": "Message('start', time=1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mido/mido", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/microdot", + "expected": "{})", + "got": "{})", + "got_raw": "{})", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/microdot", + "expected": "{'id': 'bca'})", + "got": "{'id': 'bca'})", + "got_raw": "{'id': 'bca'})", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/microdot", + "expected": "'Hello, foo!')", + "got": "'Hello, foo!')", + "got_raw": "'Hello, foo!')", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/microdot", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/microdot", + "expected": "400)", + "got": "400)", + "got_raw": "400)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/microdot", + "expected": "204)", + "got": "200)", + "got_raw": "200)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.75 + }, + { + "repo": "miguelgrinberg/microdot", + "expected": "request.g.current_user", + "got": "request.user", + "got_raw": "request.user", + "exact_match": false, + "code_bleu": 0.3567621345008163, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "miguelgrinberg/microdot", + "expected": "'None')", + "got": "'')", + "got_raw": "'')", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6 + }, + { + "repo": "miguelgrinberg/microdot", + "expected": "403)", + "got": "403)", + "got_raw": "403)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/microdot", + "expected": "'3')", + "got": "3)", + "got_raw": "3)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "miguelgrinberg/microdot", + "expected": "401)", + "got": "401)", + "got_raw": "401)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/microdot", + "expected": "{'foo': 'bar', 'baz': 'baz'})", + "got": "{'foo': 'bar', 'baz': 'baz'})", + "got_raw": "{'foo': 'bar', 'baz': 'baz'})", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/microdot", + "expected": "'POST')", + "got": "'POST')", + "got_raw": "'POST')", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "ConnectionError)", + "got": "ConnectionError)", + "got_raw": "ConnectionError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'foo'", + "got": "'foo'", + "got_raw": "'foo'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "sid", + "got": "sid", + "got_raw": "sid", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'foo'", + "got": "'foo'", + "got_raw": "'foo'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'foo'", + "got": "'foo'", + "got_raw": "'foo'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "'ev')", + "got": "io.server.emit.assert_called_with('ev', 'data', to='room', skip_sid='skip')", + "got_raw": "io.server.emit.assert_called_with('ev', 'data', to='room', skip_sid='skip')", + "exact_match": false, + "code_bleu": 0.06760229884571738, + "edit_similarity": 0.125 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "start_response)", + "got": "start_response)", + "got_raw": "start_response)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "(1, '2')", + "got": "(1, '2')", + "got_raw": "(1, '2')", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "False", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "miguelgrinberg/python-socketio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mlfoundations/open_lm", + "expected": "expected_shard_ps", + "got": "expected_shard_ps", + "got_raw": "expected_shard_ps", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mlfoundations/open_lm", + "expected": "mock_file.read()", + "got": "mock_file.read()", + "got_raw": "mock_file.read()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mlfoundations/open_lm", + "expected": "batch_size * (num_samples // batch_size)", + "got": "num_samples", + "got_raw": "num_samples", + "exact_match": false, + "code_bleu": 0.07432544468767006, + "edit_similarity": 0.43137254901960786 + }, + { + "repo": "mlfoundations/open_lm", + "expected": "\"This checkpoint was trained with a random seed of 0. Since this seed affects shard shuffling, resuming training must use the same seed.\"", + "got": "str(e) == \"All parameters are not close\"", + "got_raw": "str(e) == \"All parameters are not close\"", + "exact_match": false, + "code_bleu": 0.1561969968460128, + "edit_similarity": 0.13559322033898305 + }, + { + "repo": "mlfoundations/open_lm", + "expected": "min_shards_needed", + "got": "min_shards_needed", + "got_raw": "min_shards_needed", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "group._prompts", + "got": "group._prompts", + "got_raw": "group._prompts", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "str(exc_info.value)", + "got": "str(exc_info.value)", + "got_raw": "str(exc_info.value)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "str(excinfo.value)", + "got": "str(excinfo.value)", + "got_raw": "str(excinfo.value)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "size1", + "got": "size1", + "got_raw": "size1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "\"test\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "second", + "got": "second", + "got_raw": "second", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "\"true\"", + "got": "\"True\"", + "got_raw": "\"True\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "resource.meta[\"tags\"]", + "got": "resource.meta[\"tags\"]", + "got_raw": "resource.meta[\"tags\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "\"working\"", + "got": "\"working\"", + "got_raw": "\"working\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "task2.task_id", + "got": "task2.task_id", + "got_raw": "task2.task_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "(200, 201)", + "got": "[200, 201]", + "got_raw": "[200, 201]", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "\"2.0\"", + "got": "\"2.0\"", + "got_raw": "\"2.0\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "modelcontextprotocol/python-sdk", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mohamed-chs/convoviz", + "expected": "\"conv_123\"", + "got": "\"conv_123\"", + "got_raw": "\"conv_123\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mohamed-chs/convoviz", + "expected": "\"msg123\"", + "got": "\"msg123\"", + "got_raw": "\"msg123\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mohamed-chs/convoviz", + "expected": "{OutputKind.GRAPHS}", + "got": "[OutputKind.GRAPHS]", + "got_raw": "[OutputKind.GRAPHS]", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8947368421052632 + }, + { + "repo": "mohamed-chs/convoviz", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mohamed-chs/convoviz", + "expected": "InvalidZipError, match=\"Malicious path in ZIP\")", + "got": "InvalidZipError)", + "got_raw": "InvalidZipError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5079365079365079 + }, + { + "repo": "mohamed-chs/convoviz", + "expected": "set()", + "got": "set()", + "got_raw": "set()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mohamed-chs/convoviz", + "expected": "\"First [^1] then again [^1]\"", + "got": "\"First [1] then again [1]\"", + "got_raw": "\"First [1] then again [1]\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9629629629629629 + }, + { + "repo": "mohamed-chs/convoviz", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mohamed-chs/convoviz", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "mohamed-chs/convoviz", + "expected": "upper", + "got": "upper", + "got_raw": "upper", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mohamed-chs/convoviz", + "expected": "\"I should be deleted\"", + "got": "\"I should be deleted\"", + "got_raw": "\"I should be deleted\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mohamed-chs/convoviz", + "expected": "\"active\"", + "got": "\"alphabetical\"", + "got_raw": "\"alphabetical\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.45454545454545453 + }, + { + "repo": "mohamed-chs/convoviz", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mohamed-chs/convoviz", + "expected": "\"string_text\"", + "got": "\"string_text\"", + "got_raw": "\"string_text\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mohamed-chs/convoviz", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mohamed-chs/convoviz", + "expected": "[]", + "got": "[(\"file-999\", \"report.pdf\")]", + "got_raw": "[(\"file-999\", \"report.pdf\")]", + "exact_match": false, + "code_bleu": 0.20556680845025982, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "mootdx/mootdx", + "expected": "False)", + "got": "False)", + "got_raw": "False)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mootdx/mootdx", + "expected": "xdxr.columns", + "got": "xdxr.columns", + "got_raw": "xdxr.columns", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mootdx/mootdx", + "expected": "data)", + "got": "data)", + "got_raw": "data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mootdx/mootdx", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "mootdx/mootdx", + "expected": "'\u5e02\u573a\u4ee3\u7801\u9519\u8bef'", + "got": "'\u80a1\u7968\u4ee3\u7801\u5fc5\u987b\u662f6\u4f4d\u6570\u5b57'", + "got_raw": "'\u80a1\u7968\u4ee3\u7801\u5fc5\u987b\u662f6\u4f4d\u6570\u5b57'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.38095238095238093 + }, + { + "repo": "mootdx/mootdx", + "expected": "result.empty)", + "got": "result.empty)", + "got_raw": "result.empty)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mozillazg/python-pinyin", + "expected": "['\u5566\u5566', 'abc']", + "got": "', 'abc']", + "got_raw": "', 'abc']", + "exact_match": false, + "code_bleu": 0.28744328957046744, + "edit_similarity": 0.8181818181818182 + }, + { + "repo": "mozillazg/python-pinyin", + "expected": "[['m']]", + "got": "han, heteronym=True, style=INITIALS) == [['m']]", + "got_raw": "\n han, heteronym=True, style=INITIALS) == [['m']]", + "exact_match": false, + "code_bleu": 0.2646015952359329, + "edit_similarity": 0.25925925925925924 + }, + { + "repo": "mozillazg/python-pinyin", + "expected": "'vn'", + "got": "'lvn'", + "got_raw": "'lvn'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "mozillazg/python-pinyin", + "expected": "expect", + "got": "expect", + "got_raw": "expect", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mozillazg/python-pinyin", + "expected": "'n2'", + "got": "\u0144'", + "got_raw": "\u0144'", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "mozillazg/python-pinyin", + "expected": "['o']", + "got": "y']", + "got_raw": "y']", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.5 + }, + { + "repo": "mozillazg/python-pinyin", + "expected": "['ce4']", + "got": "ce4', 'si4']", + "got_raw": "ce4', 'si4']", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.5263157894736842 + }, + { + "repo": "mozillazg/python-pinyin", + "expected": "[['zhao1'], ['yang2']]", + "got": "han, Style.TONE3, False, 'ignore', True) == [['zhao4'], ['yang4']]", + "got_raw": "\n han, Style.TONE3, False, 'ignore', True) == [['zhao4'], ['yang4']]", + "exact_match": false, + "code_bleu": 0.13308442527111022, + "edit_similarity": 0.45454545454545453 + }, + { + "repo": "mozillazg/python-pinyin", + "expected": "[['\u00e9i', 'x\u012b', 'y\u00ec', '\u00ea\u0304', '\u1ebf', '\u00ea\u030c', '\u011bi', '\u1ec1', '\u00e8i', '\u0113i']]", + "got": "['y', 'u', 'e', 'i', 'o', 'u']", + "got_raw": "['y', 'u', 'e', 'i', 'o', 'u']", + "exact_match": false, + "code_bleu": 0.07635310154155145, + "edit_similarity": 0.28888888888888886 + }, + { + "repo": "mozillazg/python-pinyin", + "expected": "['\u4f60\u597d\u554a']", + "got": "['\u4f60\u597d\u554a']", + "got_raw": "['\u4f60\u597d\u554a']", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "m", + "got": "m", + "got_raw": "m", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "6", + "got": "5", + "got_raw": "5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "out", + "got": "out", + "got_raw": "out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "\"\"", + "got": "\"Some info\"", + "got_raw": "\"Some info\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "r.error", + "got": "r.message", + "got_raw": "r.message", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.375 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "id2", + "got": "id2", + "got_raw": "id2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "p", + "got": "p", + "got_raw": "p", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "sio", + "got": "sio", + "got_raw": "sio", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "ts", + "got": "ts", + "got_raw": "ts", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mpfaffenberger/code_puppy", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "3", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "\"Get Test\"", + "got": "\"Test endpoint\"", + "got_raw": "\"Test endpoint\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.48 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "50", + "got": "50", + "got_raw": "50", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "\"Get Test\"", + "got": "\"Test endpoint\"", + "got_raw": "\"Test endpoint\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.48 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "{}", + "got": "b\"\"", + "got_raw": "b\"\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "\"1.0.0\"", + "got": "\"1.0.0\"", + "got_raw": "\"1.0.0\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "42", + "got": "42", + "got_raw": "42", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "mr-fatalyst/fastopenapi", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nat-n/poethepoet", + "expected": "\"Hello from uv-project 0.0.99\\n\"", + "got": "\"Hello world\\n\"", + "got_raw": "\"Hello world\\n\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5106382978723404 + }, + { + "repo": "nat-n/poethepoet", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nat-n/poethepoet", + "expected": "f\"hi {arg}\\n\"", + "got": "f\"hi {arg}\\n\"", + "got_raw": "f\"hi {arg}\\n\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nat-n/poethepoet", + "expected": "\"Hello\\nWorld!\\n:)!\\n\"", + "got": "\"Hello\\nWorld!\\n:)\\n\"", + "got_raw": "\"Hello\\nWorld!\\n:)\\n\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9767441860465116 + }, + { + "repo": "nat-n/poethepoet", + "expected": "\"a1: A1, a2: A2\\nb\\n\"", + "got": "\"A1\\n\"", + "got_raw": "\n \"A1\\n\"\n \"A2\\n\"\n \"a1: A1, a2: A2\\n\"\n \"b\\n\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "nat-n/poethepoet", + "expected": "15", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "nat-n/poethepoet", + "expected": "\"parallel_with_s\u2026 | 1 going to stdout 1\\n\" \"parallel_with_s\u2026 | 4 going to stdout 4\\n\"", + "got": "result.stdout == \"\"", + "got_raw": "result.stdout == \"\"", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.2692307692307692 + }, + { + "repo": "nat-n/poethepoet", + "expected": "result.capture", + "got": "result.stdout", + "got_raw": "result.stdout", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "nat-n/poethepoet", + "expected": "[ (\"abcx'[!]\", True), (\"]'ydef\", False), (\"*z\", True), (\"[foo\", False), (\"?\", True), (\"?\", True), ]", + "got": "[(\"abc\", False), (\"def\", False), (\"*s\", True), (\"p\", False), (\"a\", False), (\"c\", False), (\"e?\", True)]", + "got_raw": "[(\"abc\", False), (\"def\", False), (\"*s\", True), (\"p\", False), (\"a\", False), (\"c\", False), (\"e?\", True)]", + "exact_match": false, + "code_bleu": 0.36721213570787975, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "nat-n/poethepoet", + "expected": "\"Poe => poe_test_echo here we go...\\n\" \"Poe => :\\n\" \"Poe <= poe_test_echo about\\n\" \"Poe <= poe_test_echo hello\\n\" \"Poe ?? unresolved dependency task results via uses option for task 'think'\\n\" \"Poe ?? unresolved dependency task results via uses option for task\" \" 'deep-graph-with-args'\\n\"", + "got": "result.stdout", + "got_raw": "result.stdout", + "exact_match": false, + "code_bleu": 0.16282193263962697, + "edit_similarity": 0.006622516556291391 + }, + { + "repo": "nat-n/poethepoet", + "expected": "\"Hello there!\\n\"", + "got": "\"test\\n\"", + "got_raw": "\"test\\n\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "nat-n/poethepoet", + "expected": "\"deploying to admin:12345@dev.example.com:8080\\n\"", + "got": "\"deploying to admin:12345@dev.example.com:8080\"", + "got_raw": "\"deploying to admin:12345@dev.example.com:8080\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9791666666666666 + }, + { + "repo": "nat-n/poethepoet", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nat-n/poethepoet", + "expected": "\"cmd task with verbosity -1 -1\\n\"", + "got": "\"cmd task with verbosity -1 0\\n\"", + "got_raw": "\"cmd task with verbosity -1 0\\n\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9538461538461539 + }, + { + "repo": "nat-n/poethepoet", + "expected": "\"parallel_of_seq\u2026 | seq1\\n\" \"parallel_of_seq\u2026 | seq2\\n\" \"parallel_of_seq\u2026 | para1\\n\"", + "got": "result.exit_code", + "got_raw": "result.exit_code", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.16 + }, + { + "repo": "nat-n/poethepoet", + "expected": "\"build+test\"", + "got": "\"build+test\"", + "got_raw": "\"build+test\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nat-n/poethepoet", + "expected": "\"Great success!\\nfailing fast with error\\n\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.08888888888888889 + }, + { + "repo": "nat-n/poethepoet", + "expected": "\"ok\"", + "got": "\"ok\"", + "got_raw": "\"ok\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nat-n/poethepoet", + "expected": "\"Poe <= echo A1\\n\" \"Poe <= echo A2\\n\" \"Poe => 'a1: ' + ${a1} + ', a2: ' + ${a2}\\n\" \"Poe => echo b\\n\"", + "got": "result.stdout", + "got_raw": "result.stdout", + "exact_match": false, + "code_bleu": 0.2849383821193472, + "edit_similarity": 0.035398230088495575 + }, + { + "repo": "nat-n/poethepoet", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nbQA-dev/nbQA", + "expected": "expected_err", + "got": "expected_err", + "got_raw": "expected_err", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nbQA-dev/nbQA", + "expected": "err", + "got": "err", + "got_raw": "err", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nbQA-dev/nbQA", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nbQA-dev/nbQA", + "expected": "err", + "got": "err", + "got_raw": "err", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nbQA-dev/nbQA", + "expected": "123", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "nbQA-dev/nbQA", + "expected": "expected_out", + "got": "expected_out", + "got_raw": "expected_out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nbQA-dev/nbQA", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nbQA-dev/nbQA", + "expected": "expected_out.replace( WRONG_EXAMPLE_NOTEBOOK, os.path.abspath(WRONG_EXAMPLE_NOTEBOOK) )", + "got": "expected_out.replace(WRONG_EXAMPLE_NOTEBOOK, \"\")", + "got_raw": "expected_out.replace(WRONG_EXAMPLE_NOTEBOOK, \"\")", + "exact_match": false, + "code_bleu": 0.40033272200018105, + "edit_similarity": 0.6814814814814815 + }, + { + "repo": "nbQA-dev/nbQA", + "expected": "expected.replace(\".md\", \".ipynb\")", + "got": "expected", + "got_raw": "expected", + "exact_match": false, + "code_bleu": 0.07432544468767006, + "edit_similarity": 0.3902439024390244 + }, + { + "repo": "nerfbaselines/nerfbaselines", + "expected": "render", + "got": "render", + "got_raw": "render", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nerfbaselines/nerfbaselines", + "expected": "[4, 4, 9, 9, 12, 12]", + "got": "12", + "got_raw": "12", + "exact_match": false, + "code_bleu": 0.04573873519241235, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "nerfbaselines/nerfbaselines", + "expected": "f'{get_resources_utilization_info.__module__}:{get_resources_utilization_info.__name__}'", + "got": "callback", + "got_raw": "callback", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.10416666666666667 + }, + { + "repo": "nerfbaselines/nerfbaselines", + "expected": "json.dumps(round_floats(orig), indent=2, sort_keys=True)", + "got": "round_floats(data2)", + "got_raw": "round_floats(data2)", + "exact_match": false, + "code_bleu": 0.11956064662461523, + "edit_similarity": 0.4533333333333333 + }, + { + "repo": "nerfbaselines/nerfbaselines", + "expected": "arr.shape", + "got": "arr.shape", + "got_raw": "arr.shape", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nerfbaselines/nerfbaselines", + "expected": "members", + "got": "members", + "got_raw": "members", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nerfbaselines/nerfbaselines", + "expected": "FileExistsError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5925925925925926 + }, + { + "repo": "nerfbaselines/nerfbaselines", + "expected": "\"docker\"", + "got": "\"docker\"", + "got_raw": "\"docker\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nerfbaselines/nerfbaselines", + "expected": "np.float32", + "got": "uv.dtype", + "got_raw": "uv.dtype", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "nerfbaselines/nerfbaselines", + "expected": "\"Hello world\"", + "got": "\"Hello world\"", + "got_raw": "\"Hello world\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nerfbaselines/nerfbaselines", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nerfbaselines/nerfbaselines", + "expected": "np.uint8", + "got": "np.uint8", + "got_raw": "np.uint8", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nerfbaselines/nerfbaselines", + "expected": "info", + "got": "info", + "got_raw": "info", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nipunsadvilkar/pySBD", + "expected": "\"char_span must be False if clean is True. \"\\ \"Since `clean=True` will modify original text.\"", + "got": "\"Both clean and char_span are set to True\"", + "got_raw": "\"Both clean and char_span are set to True\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.32592592592592595 + }, + { + "repo": "nipunsadvilkar/pySBD", + "expected": "expected_sents", + "got": "expected_sents", + "got_raw": "expected_sents", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nipunsadvilkar/pySBD", + "expected": "expected_sents", + "got": "expected_sents", + "got_raw": "expected_sents", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nipunsadvilkar/pySBD", + "expected": "expected_sents", + "got": "expected_sents", + "got_raw": "expected_sents", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nipunsadvilkar/pySBD", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nonebot/nonebot2", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nonebot/nonebot2", + "expected": "kws", + "got": "kws", + "got_raw": "kws", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nonebot/nonebot2", + "expected": "r\"\\white\\\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nonebot/nonebot2", + "expected": "'\"123\"'", + "got": "'\"123\"'", + "got_raw": "'\"123\"'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nonebot/nonebot2", + "expected": "escape_text(\"[fake:image]Is Bad\")", + "got": "\"Is Bad\"", + "got_raw": "\"Is Bad\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.3902439024390244 + }, + { + "repo": "nonebot/nonebot2", + "expected": "TypeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "nonebot/nonebot2", + "expected": "\"nested\"", + "got": "\"plugins.nested\"", + "got_raw": "\"plugins.nested\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "nonebot/nonebot2", + "expected": "pattern", + "got": "re.compile(pattern)", + "got_raw": "re.compile(pattern)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.5384615384615384 + }, + { + "repo": "nonebot/nonebot2", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nonebot/nonebot2", + "expected": "bot", + "got": "bot", + "got_raw": "bot", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nonebot/nonebot2", + "expected": "FakeMessage([pic1, pic2, pic3])", + "got": "pic1", + "got_raw": "pic1", + "exact_match": false, + "code_bleu": 0.05946035575013606, + "edit_similarity": 0.22857142857142856 + }, + { + "repo": "nonebot/nonebot2", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nonebot/nonebot2", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nonebot/nonebot2", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nonebot/nonebot2", + "expected": "ValueError, match=\"private attribute\")", + "got": "AttributeError)", + "got_raw": "AttributeError)", + "exact_match": false, + "code_bleu": 0.17909498863725698, + "edit_similarity": 0.33962264150943394 + }, + { + "repo": "nonebot/nonebot2", + "expected": "ValidationError)", + "got": "ValidationError)", + "got_raw": "ValidationError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nschloe/meshio", + "expected": "ref_sum_cell_tags", + "got": "ref_sum_cell_tags", + "got_raw": "ref_sum_cell_tags", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nschloe/meshio", + "expected": "ref_num_cells", + "got": "ref_num_cells[mesh.cells[0].type]", + "got_raw": "ref_num_cells[mesh.cells[0].type]", + "exact_match": false, + "code_bleu": 0.11390778025531027, + "edit_similarity": 0.5652173913043478 + }, + { + "repo": "nschloe/meshio", + "expected": "{\"a\": {\"triangle\": [0.5, 1.3]}})", + "got": "{\"a\": [[0.5], [1.3]]})", + "got_raw": "{\"a\": [[0.5], [1.3]]})", + "exact_match": false, + "code_bleu": 0.27073362211548463, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "nschloe/meshio", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nschloe/meshio", + "expected": "(8, 3)", + "got": "(12, 3)", + "got_raw": "(12, 3)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "nschloe/meshio", + "expected": "{\"grain0-grain1\": [[0, 0, 1, 1, 1]]})", + "got": "{})", + "got_raw": "{})", + "exact_match": false, + "code_bleu": 0.10649051737437873, + "edit_similarity": 0.15 + }, + { + "repo": "nschloe/meshio", + "expected": "mesh.cells[ic].type", + "got": "mesh.cells[ic].type", + "got_raw": "mesh.cells[ic].type", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nschloe/meshio", + "expected": "tol * abs(ref_sum)", + "got": "tol * ref_sum", + "got_raw": "tol * ref_sum", + "exact_match": false, + "code_bleu": 0.3194715521231362, + "edit_similarity": 0.8387096774193549 + }, + { + "repo": "nschloe/meshio", + "expected": "\"exodus\"", + "got": "\"vtkm\"", + "got_raw": "\"vtkm\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "nschloe/tikzplotlib", + "expected": "__file__[:-3] + \"_reference.tex\")", + "got": "\"test_plot_reference.tex\")", + "got_raw": "\"test_plot_reference.tex\")", + "exact_match": false, + "code_bleu": 0.11939665909150465, + "edit_similarity": 0.576271186440678 + }, + { + "repo": "nschloe/tikzplotlib", + "expected": "\"test_horizontal_alignment_reference.tex\")", + "got": "\"test_bar_labels.tex\")", + "got_raw": "\"test_bar_labels.tex\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.46875 + }, + { + "repo": "nschloe/tikzplotlib", + "expected": "\"test_line_set_data_reference.tex\")", + "got": "\"test_line_basic_reference.tex\")", + "got_raw": "\"test_line_basic_reference.tex\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.835820895522388 + }, + { + "repo": "nschloe/tikzplotlib", + "expected": "tikzs[0]", + "got": "tikzs[0]", + "got_raw": "tikzs[0]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nschloe/tikzplotlib", + "expected": "\"test_legend_best_location_reference.tex\")", + "got": "\"test_legend_best_location_rst.rst\")", + "got_raw": "\"test_legend_best_location_rst.rst\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7948717948717948 + }, + { + "repo": "nschloe/tikzplotlib", + "expected": "26", + "got": "100", + "got_raw": "100", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "nschloe/tikzplotlib", + "expected": "__file__[:-3] + \"_reference.tex\")", + "got": "\"semilogy_test.py\")", + "got_raw": "\"semilogy_test.py\")", + "exact_match": false, + "code_bleu": 0.11939665909150465, + "edit_similarity": 0.2692307692307692 + }, + { + "repo": "nschloe/tikzplotlib", + "expected": "\"test_basic_sin_reference.tex\")", + "got": "\"test_plot.tex\")", + "got_raw": "\"test_plot.tex\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5106382978723404 + }, + { + "repo": "ntno/mkdocs-terminal", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ntno/mkdocs-terminal", + "expected": "\"pluglets/tile_grid/templates/j2-partials/tiles.html\")", + "got": "\"pluglets/tile_grid/templates/j2-partials/tiles.html\")", + "got_raw": "\"pluglets/tile_grid/templates/j2-partials/tiles.html\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ntno/mkdocs-terminal", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ntno/mkdocs-terminal", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ntno/mkdocs-terminal", + "expected": "rendered_nav_search_button", + "got": "rendered_nav_search_button", + "got_raw": "rendered_nav_search_button", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ntno/mkdocs-terminal", + "expected": "f\"Palette '{palette_name}': Font color\")", + "got": "get pytest.mark.parametrize(\"component\", colors)", + "got_raw": "get pytest.mark.parametrize(\"component\", colors)", + "exact_match": false, + "code_bleu": 0.11390778025531027, + "edit_similarity": 0.4318181818181818 + }, + { + "repo": "ntno/mkdocs-terminal", + "expected": "rendered_grid)", + "got": "rendered_grid )", + "got_raw": " rendered_grid )", + "exact_match": false, + "code_bleu": 0.7071067811865476, + "edit_similarity": 0.9655172413793104 + }, + { + "repo": "ntno/mkdocs-terminal", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nvbn/thefuck", + "expected": "known_args)", + "got": "known_args)", + "got_raw": "known_args)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nvbn/thefuck", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nvbn/thefuck", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nvbn/thefuck", + "expected": "OSError)", + "got": "OSError)", + "got_raw": "OSError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nvbn/thefuck", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nvbn/thefuck", + "expected": "'ls || cd'", + "got": "'ls || cd'", + "got_raw": "'ls || cd'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nvbn/thefuck", + "expected": "[result]", + "got": "result", + "got_raw": "result", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "nvbn/thefuck", + "expected": "Command(called, '')", + "got": "Command('sudo ls', 'sudo ls')", + "got_raw": "Command('sudo ls', 'sudo ls')", + "exact_match": false, + "code_bleu": 0.3303164318013807, + "edit_similarity": 0.5833333333333334 + }, + { + "repo": "nvbn/thefuck", + "expected": "'Bash 3.5.9'", + "got": "'3.5.9'", + "got_raw": "'3.5.9'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "nvbn/thefuck", + "expected": "{CorrectedCommand('ls')}", + "got": "CorrectedCommand('ls', None, 100)}", + "got_raw": " CorrectedCommand('ls', None, 100)}", + "exact_match": false, + "code_bleu": 0.31239399369202553, + "edit_similarity": 0.7931034482758621 + }, + { + "repo": "nvbn/thefuck", + "expected": "new_cmd", + "got": "new_cmd", + "got_raw": "new_cmd", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nvbn/thefuck", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nvbn/thefuck", + "expected": "r\"sed -e 's/\\/foo/bar/'\"", + "got": "_new_command(Command('sed -e s/foo/bar/ -e s/foo/bar', sed_unterminated_s))", + "got_raw": "_new_command(Command('sed -e s/foo/bar/ -e s/foo/bar', sed_unterminated_s))", + "exact_match": false, + "code_bleu": 0.11868405219520975, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "nvbn/thefuck", + "expected": "new_command", + "got": "new_command", + "got_raw": "new_command", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nvbn/thefuck", + "expected": "['test']", + "got": "['test']", + "got_raw": "['test']", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nvbn/thefuck", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nvbn/thefuck", + "expected": "new_command", + "got": "new_command", + "got_raw": "new_command", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nvbn/thefuck", + "expected": "output", + "got": "output", + "got_raw": "output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nvbn/thefuck", + "expected": "['ls', 'rm']", + "got": "['ls', 'rm']", + "got_raw": "['ls', 'rm']", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "nvbn/thefuck", + "expected": "after", + "got": "after", + "got_raw": "after", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "offu/WeRoBot", + "expected": "'toUser'", + "got": "message", + "got_raw": "message", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "offu/WeRoBot", + "expected": "u'\u606d\u559c\u8131\u5751'", + "got": "'\u606d\u559c\u8131\u5751'", + "got_raw": "'\u606d\u559c\u8131\u5751'", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "offu/WeRoBot", + "expected": "xml", + "got": "xml", + "got_raw": "xml", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "offu/WeRoBot", + "expected": "[(scan_handler, 0)]", + "got": "[(scan_handler, 0)]", + "got_raw": "[(scan_handler, 0)]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "offu/WeRoBot", + "expected": "getattr(logging, level.upper())", + "got": "getattr(logging, level.upper())", + "got_raw": "getattr(logging, level.upper())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "offu/WeRoBot", + "expected": "\"hq\"", + "got": "hq", + "got_raw": "hq", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "offu/WeRoBot", + "expected": "u'\u626b\u63cf\u63a8\u9001'", + "got": "'\u626b\u63cf\u63a8\u9001'", + "got_raw": "'\u626b\u63cf\u63a8\u9001'", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "offu/WeRoBot", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "offu/WeRoBot", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "offu/WeRoBot", + "expected": "\"url\"", + "got": "\"hq\"", + "got_raw": "\"hq\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "offu/WeRoBot", + "expected": "token", + "got": "token", + "got_raw": "token", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "cfg.jsonnet[\"param\"]", + "got": "cfg[\"param\"]", + "got_raw": "cfg[\"param\"]", + "exact_match": false, + "code_bleu": 0.4714045207910316, + "edit_similarity": 0.75 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "path2", + "got": "path2", + "got_raw": "path2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "cfg.a1", + "got": "cfg.data_field_factory", + "got_raw": "cfg.data_field_factory", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.35714285714285715 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "torch.Tensor", + "got": "torch.Tensor", + "got_raw": "torch.Tensor", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "cfg[\"op1\"]", + "got": "cfg[\"op1\"]", + "got_raw": "cfg[\"op1\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "\"required\"", + "got": "\"required\"", + "got_raw": "\"required\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "Namespace()", + "got": "Namespace()", + "got_raw": "Namespace()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "[1, 2]", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "cfg.data", + "got": "cfg.data", + "got_raw": "cfg.data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "\"B\"", + "got": "\"b\"", + "got_raw": "\"b\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "json.loads(help_schema)", + "got": "json.loads(help_schema)", + "got_raw": "json.loads(help_schema)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "loaders[\"omegaconf\"]", + "got": "yaml_dump", + "got_raw": "yaml_dump", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.13793103448275862 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "\"secret\"", + "got": "\"secret\"", + "got_raw": "\"secret\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "TypeError)", + "got": "ArgumentError)", + "got_raw": "ArgumentError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5833333333333334 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "\"-\"", + "got": "sys.stdout", + "got_raw": "sys.stdout", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "origins", + "got": "origins", + "got_raw": "origins", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "\"WithLogger\"", + "got": "\"test_logger\"", + "got_raw": "\"test_logger\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.64 + }, + { + "repo": "omni-us/jsonargparse", + "expected": "cfg", + "got": "cfg", + "got_raw": "cfg", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "online-judge-tools/oj", + "expected": "len(expected))", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.2149139863647084, + "edit_similarity": 0.125 + }, + { + "repo": "online-judge-tools/oj", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "online-judge-tools/oj", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "online-judge-tools/oj", + "expected": "expect['data'])", + "got": "expect['data'])", + "got_raw": "expect['data'])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "oomol-lab/epub-translator", + "expected": "rt.text", + "got": "rt.text", + "got_raw": "rt.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "oomol-lab/epub-translator", + "expected": "spine_names", + "got": "spine_names", + "got_raw": "spine_names", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "oomol-lab/epub-translator", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "oomol-lab/epub-translator", + "expected": "result_str)", + "got": "result_str)", + "got_raw": "result_str)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "oomol-lab/epub-translator", + "expected": "10000", + "got": "10000", + "got_raw": "10000", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "oomol-lab/epub-translator", + "expected": "\"\"", + "got": "\"Test\"", + "got_raw": "\"Test\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "oomol-lab/epub-translator", + "expected": "r\"\\sqrt{x}\"", + "got": "\"\\\\sqrt{x}\"", + "got_raw": "\"\\\\sqrt{x}\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "oomol-lab/epub-translator", + "expected": "\"\u5e94\u8be5\u5305\u542b\u7236\u5143\u7d20\u7684\u8bd1\u6587\")", + "got": "\u5143\u7d20\")", + "got_raw": "\u5143\u7d20\")", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "oomol-lab/epub-translator", + "expected": "5)", + "got": "3)", + "got_raw": "3)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "collection_size_callback_factory(collection, 3))", + "got": "collection_size_callback_factory(collection, 3) )", + "got_raw": " collection_size_callback_factory(collection, 3) )", + "exact_match": false, + "code_bleu": 1.0, + "edit_similarity": 0.9896907216494846 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "len(query_results)", + "got": "len(query_results)", + "got_raw": "len(query_results)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "created_at.data_type", + "got": "created_at.type", + "got_raw": "created_at.type", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "\"first-doc-5\"", + "got": "\"first-doc-5\"", + "got_raw": "\"first-doc-5\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "'mongodb'", + "got": "\"mongodb\"", + "got_raw": "\"mongodb\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "sum( len(v) for v in document_chunks.values() )", + "got": "len(document_chunks)", + "got_raw": "len(document_chunks)", + "exact_match": false, + "code_bleu": 0.14327599090980558, + "edit_similarity": 0.5970149253731343 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "new_store.col.num_entities", + "got": "new_store.col.num_entities", + "got_raw": "new_store.col.num_entities", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "f\"{field} eq 'test''_{field}'\"", + "got": "f\"{field} eq '{needs_escaping_value}'\"", + "got_raw": "f\"{field} eq '{needs_escaping_value}'\"", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6176470588235294 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "\"The quick brown fox jumped over the slimy green toad.\"", + "got": "\"The quick brown fox jumped over the slimy green toad.\"", + "got_raw": "\"The quick brown fox jumped over the slimy green toad.\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "openai/chatgpt-retrieval-plugin", + "expected": "\"doc:first-doc:chunk:first-doc-4\"", + "got": "\"first-doc-4\"", + "got_raw": "\"first-doc-4\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5652173913043478 + }, + { + "repo": "openapi-generators/openapi-python-client", + "expected": "MyModel(enum_prop=2))", + "got": "{\"enumProp\": 2})", + "got_raw": "{\"enumProp\": 2})", + "exact_match": false, + "code_bleu": 0.2095384381863861, + "edit_similarity": 0.4864864864864865 + }, + { + "repo": "openapi-generators/openapi-python-client", + "expected": "MyModel(date_prop=date_value))", + "got": "date_value)", + "got_raw": "date_value)", + "exact_match": false, + "code_bleu": 0.20203050891044216, + "edit_similarity": 0.5365853658536586 + }, + { + "repo": "openapi-generators/openapi-python-client", + "expected": "KeyError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "openapi-generators/openapi-python-client", + "expected": "DerivedModel(req1=\"a\", req2=\"b\", req3=\"c\", opt=\"d\"))", + "got": "MyModel(req1=\"a\", req2=\"b\"))", + "got_raw": "MyModel(req1=\"a\", req2=\"b\"))", + "exact_match": false, + "code_bleu": 0.4453034863393708, + "edit_similarity": 0.65 + }, + { + "repo": "openapi-generators/openapi-python-client", + "expected": "\"MyEnum\")", + "got": "optional_type)", + "got_raw": "optional_type)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.17391304347826086 + }, + { + "repo": "openapi-generators/openapi-python-client", + "expected": "\"int | Unset\")", + "got": "\"int | Unset\")", + "got_raw": "\"int | Unset\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "openapi-generators/openapi-python-client", + "expected": "ModelWithPrefixItems(array_prop=[SimpleObject(name=\"a\"), \"b\"]))", + "got": "ModelWithPrefixItems)", + "got_raw": "ModelWithPrefixItems)", + "exact_match": false, + "code_bleu": 0.07432544468767006, + "edit_similarity": 0.5 + }, + { + "repo": "openapi-generators/openapi-python-client", + "expected": "MyModel( boolean_prop=True, string_prop=\"a\", number_prop=1.5, int_prop=2, any_object_prop=AnyObject(boolean_prop=False), null_prop=None, any_prop=\"e\", ))", + "got": "MyModel(json_data))", + "got_raw": "MyModel(json_data))", + "exact_match": false, + "code_bleu": 0.06105418943309763, + "edit_similarity": 0.1744186046511628 + }, + { + "repo": "openapi-generators/openapi-python-client", + "expected": "\"list[Any] | Unset\")", + "got": "\"Any\")", + "got_raw": "\"Any\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.46153846153846156 + }, + { + "repo": "openatx/uiautomator2", + "expected": "d(text=\"Status Bar\").wait(timeout=5))", + "got": "d.watcher.running())", + "got_raw": "d.watcher.running())", + "exact_match": false, + "code_bleu": 0.14342069282273692, + "edit_similarity": 0.3157894736842105 + }, + { + "repo": "openatx/uiautomator2", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "openatx/uiautomator2", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "openatx/uiautomator2", + "expected": "\"hello world\")", + "got": "\"hello world\")", + "got_raw": "\"hello world\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "openatx/uiautomator2", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "openatx/uiautomator2", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "openatx/uiautomator2", + "expected": "(193, 321, 3)", + "got": "im_ae86.shape", + "got_raw": "im_ae86.shape", + "exact_match": false, + "code_bleu": 0.16282193263962697, + "edit_similarity": 0.0 + }, + { + "repo": "openatx/uiautomator2", + "expected": "time_used", + "got": "time_used", + "got_raw": "time_used", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "openatx/uiautomator2", + "expected": "\"3\"", + "got": "'7'", + "got_raw": "'7'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "openatx/uiautomator2", + "expected": "2 + 3", + "got": "4", + "got_raw": "4", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.0 + }, + { + "repo": "openatx/uiautomator2", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "orbingol/NURBS-Python", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "orbingol/NURBS-Python", + "expected": "degree_new", + "got": "degree_new", + "got_raw": "degree_new", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "orbingol/NURBS-Python", + "expected": "TypeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "orbingol/NURBS-Python", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "orbingol/NURBS-Python", + "expected": "val", + "got": "val", + "got_raw": "val", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "orbingol/NURBS-Python", + "expected": "c_weights", + "got": "c_weights", + "got_raw": "c_weights", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "gold_paragraphs_df.columns", + "got": "gold_paragraphs_df.columns", + "got_raw": "gold_paragraphs_df.columns", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "oughtinc/ice", + "expected": "[0, 1, 2]", + "got": "F(10)", + "got_raw": "F(10)", + "exact_match": false, + "code_bleu": 0.17265002270610103, + "edit_similarity": 0.14285714285714285 + }, + { + "repo": "oughtinc/ice", + "expected": "0.99", + "got": "0.9", + "got_raw": "0.9", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "oughtinc/ice", + "expected": "\"The proportion of children whose census status was recorded as moved or unknown did not differ significantly between the groups (P=0.71 and P=0.36, respectively).\"", + "got": "sentences[1]", + "got_raw": "sentences[1]", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.09090909090909091 + }, + { + "repo": "oughtinc/ice", + "expected": "{\"things\": [1, 2, 3]}", + "got": "[1, 2, 3]", + "got_raw": "[1, 2, 3]", + "exact_match": false, + "code_bleu": 0.6363636363636364, + "edit_similarity": 0.6 + }, + { + "repo": "oughtinc/ice", + "expected": "{\"name\": \"world\"}", + "got": "[\"hello world\"]", + "got_raw": "[\"hello world\"]", + "exact_match": false, + "code_bleu": 0.22795070569547776, + "edit_similarity": 0.5625 + }, + { + "repo": "oughtinc/ice", + "expected": "\"Foo\"", + "got": "{\"a\": \"Foo\"}", + "got_raw": "{\"a\": \"Foo\"}", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.5882352941176471 + }, + { + "repo": "pajbot/pajbot", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pajbot/pajbot", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pajbot/pajbot", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pajbot/pajbot", + "expected": "clean_up_message(\" \")", + "got": "clean_up_message(\" \\U000e0000 \")", + "got_raw": "clean_up_message(\" \\U000e0000 \")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7636363636363637 + }, + { + "repo": "pajbot/pajbot", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pajbot/pajbot", + "expected": "filter_message(\" forsen\")", + "got": "filter_message(\"forsen\")", + "got_raw": "filter_message(\"forsen\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.9795918367346939 + }, + { + "repo": "palantir/python-language-server", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "palantir/python-language-server", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "palantir/python-language-server", + "expected": "'flake8'", + "got": "'flake8'", + "got_raw": "'flake8'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "palantir/python-language-server", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "palantir/python-language-server", + "expected": "lsp.DiagnosticSeverity.Error", + "got": "lsp.DiagnosticSeverity.Error", + "got_raw": "lsp.DiagnosticSeverity.Error", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "palantir/python-language-server", + "expected": "[ mock.call(1), mock.call(2), mock.call(3), ])", + "got": "[mock.call(1), mock.call(2), mock.call(3)])", + "got_raw": "[mock.call(1), mock.call(2), mock.call(3)])", + "exact_match": false, + "code_bleu": 0.9025592277146132, + "edit_similarity": 0.9662921348314607 + }, + { + "repo": "palantir/python-language-server", + "expected": "'D100: Missing docstring in public module'", + "got": "'W101 Missing docstring in public module'", + "got_raw": "'W101 Missing docstring in public module'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9397590361445783 + }, + { + "repo": "palantir/python-language-server", + "expected": "doc.uri", + "got": "DOC_URI", + "got_raw": "DOC_URI", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.0 + }, + { + "repo": "palantir/python-language-server", + "expected": "{'line': 0, 'character': 0}", + "got": "{'line': 0, 'character': 0}", + "got_raw": "{'line': 0, 'character': 0}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "palantir/python-language-server", + "expected": "'datetime'", + "got": "'date('", + "got_raw": "'date('", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "palantir/python-language-server", + "expected": "'W191'", + "got": "'W191'", + "got_raw": "'W191'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "palantir/python-language-server", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "palantir/python-language-server", + "expected": "{ 'line': 0, 'character': 0, }", + "got": "{'line': 0, 'character': 0}", + "got_raw": "{'line': 0, 'character': 0}", + "exact_match": false, + "code_bleu": 0.8132418032488603, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "palantir/python-language-server", + "expected": "[ \"def hello(a, b):\\n\", \" print a, b\\n\" ]", + "got": "old", + "got_raw": "old", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.045454545454545456 + }, + { + "repo": "pallets/quart", + "expected": "ValueError)", + "got": "Exception)", + "got_raw": "Exception)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "pallets/quart", + "expected": "\"/\"", + "got": "\"/\"", + "got_raw": "\"/\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pallets/quart", + "expected": "205", + "got": "205", + "got_raw": "205", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pallets/quart", + "expected": "500", + "got": "500", + "got_raw": "500", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pallets/quart", + "expected": "b\"43\"", + "got": "b\"43\"", + "got_raw": "b\"43\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pallets/quart", + "expected": "await response.get_data(as_text=True)", + "got": "await response.get_json())", + "got_raw": "await response.get_json())", + "exact_match": false, + "code_bleu": 0.2975800657105459, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pallets/quart", + "expected": "\"/path\"", + "got": "\"/path\"", + "got_raw": "\"/path\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pallets/quart", + "expected": "\"bar\"", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pallets/quart", + "expected": "interface.get_cookie_domain(app) or \"\"", + "got": "interface.get_cookie_domain(app) or \"\")", + "got_raw": "interface.get_cookie_domain(app) or \"\")", + "exact_match": false, + "code_bleu": 0.8801117367933934, + "edit_similarity": 0.987012987012987 + }, + { + "repo": "pallets/quart", + "expected": "await body", + "got": "await body.read()", + "got_raw": "await body.read()", + "exact_match": false, + "code_bleu": 0.2907153684841096, + "edit_similarity": 0.7407407407407407 + }, + { + "repo": "pallets/quart", + "expected": "b\"GET /\"", + "got": "b\"GET /\"", + "got_raw": "b\"GET /\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pallets/quart", + "expected": "b\"42\"", + "got": "b\"42\"", + "got_raw": "b\"42\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pallets/quart", + "expected": "result.output", + "got": "result.output", + "got_raw": "result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pallets/quart", + "expected": "\"/bob/post\"", + "got": "\"/bob/post\"", + "got_raw": "\"/bob/post\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pallets/quart", + "expected": "404", + "got": "404", + "got_raw": "404", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pallets/quart", + "expected": "\"bar FOO\"", + "got": "\"bar FOO\"", + "got_raw": "\"bar FOO\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pallets/quart", + "expected": "[1, 2]", + "got": "[1, 2]", + "got_raw": "[1, 2]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "ret)", + "got": "ret)", + "got_raw": "ret)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "'test_4')", + "got": "cluster.get_slot_name_on_primary(self.p.name, stream_node))", + "got_raw": "cluster.get_slot_name_on_primary(self.p.name, stream_node))", + "exact_match": false, + "code_bleu": 0.09552040806823771, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "patroni/patroni", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "20)", + "got": "20)", + "got_raw": "20)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "conn.on_role_change(\"primary\"))", + "got": "conn.is_aws)", + "got_raw": "conn.is_aws)", + "exact_match": false, + "code_bleu": 0.29036157037406557, + "edit_similarity": 0.37209302325581395 + }, + { + "repo": "patroni/patroni", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "0o123)", + "got": "'0123')", + "got_raw": "'0123')", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "patroni/patroni", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "SystemExit)", + "got": "AuthFailed)", + "got_raw": "AuthFailed)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.2727272727272727 + }, + { + "repo": "patroni/patroni", + "expected": "2)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "patroni/patroni", + "expected": "errors)", + "got": "errors)", + "got_raw": "errors)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "wd.disable())", + "got": "wd.deactivate())", + "got_raw": "wd.deactivate())", + "exact_match": false, + "code_bleu": 0.488923022434901, + "edit_similarity": 0.6206896551724138 + }, + { + "repo": "patroni/patroni", + "expected": "cmd)", + "got": "cmd)", + "got_raw": "cmd)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "5)", + "got": "5)", + "got_raw": "5)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "Real)", + "got": "Real)", + "got_raw": "Real)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "'patroni')", + "got": "'patroni')", + "got_raw": "'patroni')", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "patroni/patroni", + "expected": "EtcdError)", + "got": "DCSError)", + "got_raw": "DCSError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.631578947368421 + }, + { + "repo": "patroni/patroni", + "expected": "'/tmp')", + "got": "self.p.host)", + "got_raw": "self.p.host)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.21052631578947367 + }, + { + "repo": "pdm-project/pdm", + "expected": "original", + "got": "original", + "got_raw": "original", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pdm-project/pdm", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pdm-project/pdm", + "expected": "config", + "got": "config", + "got_raw": "config", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pdm-project/pdm", + "expected": "is_quiet", + "got": "is_quiet", + "got_raw": "is_quiet", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pdm-project/pdm", + "expected": "scripts", + "got": "scripts", + "got_raw": "scripts", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pdm-project/pdm", + "expected": "{\"default\", \"extra1\", \"extra2\", \"all\"}", + "got": "[\"default\", \"extra1\", \"extra2\", \"all\"]", + "got_raw": "[\"default\", \"extra1\", \"extra2\", \"all\"]", + "exact_match": false, + "code_bleu": 0.7598356856515925, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "pdm-project/pdm", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pdm-project/pdm", + "expected": "\"3.0.1\"", + "got": "\"3.0.1\"", + "got_raw": "\"3.0.1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pdm-project/pdm", + "expected": "result.stderr", + "got": "result.output", + "got_raw": "result.output", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "pdm-project/pdm", + "expected": "\"desc\"", + "got": "\"desc\"", + "got_raw": "\"desc\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pdm-project/pdm", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pdm-project/pdm", + "expected": "Version(\"3.8.0\")", + "got": "Version(\"3.8.0\")", + "got_raw": "Version(\"3.8.0\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pdm-project/pdm", + "expected": "\"2019.3\"", + "got": "\"2019.3\"", + "got_raw": "\"2019.3\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pdm-project/pdm", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pdm-project/pdm", + "expected": "\"bar\"", + "got": "\"bar\"", + "got_raw": "\"bar\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pdm-project/pdm", + "expected": "cache", + "got": "cache._cache", + "got_raw": "cache._cache", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.5882352941176471 + }, + { + "repo": "pdm-project/pdm", + "expected": "list(case)", + "got": "list(case)", + "got_raw": "list(case)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pdm-project/pdm", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pdm-project/pdm", + "expected": "\"2.7\"", + "got": "\"idna==2.8\"", + "got_raw": "\"idna==2.8\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "pdm-project/pdm", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "langs[1]", + "got": "langs[1]", + "got_raw": "langs[1]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "out", + "got": "out", + "got_raw": "out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "cfg.extensions", + "got": "cfg.extensions", + "got_raw": "cfg.extensions", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "5", + "got": "(1, 1)", + "got_raw": "(1, 1)", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "e", + "got": "e", + "got_raw": "e", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "5", + "got": "6", + "got_raw": "6", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "ids", + "got": "ids", + "got_raw": "ids", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "err", + "got": "err", + "got_raw": "err", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "peteromallet/desloppify", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "petl-developers/petl", + "expected": "a.baz[0])", + "got": "a.baz[0])", + "got_raw": "a.baz[0])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "petl-developers/petl", + "expected": "StopIteration)", + "got": "StopIteration)", + "got_raw": "StopIteration)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "petl-developers/petl", + "expected": "a['baz'][2])", + "got": "a['baz'][2])", + "got_raw": "a['baz'][2])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "petl-developers/petl", + "expected": "('spong', 'bar')", + "got": "('spong', 'bar')", + "got_raw": "('spong', 'bar')", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "petl-developers/petl", + "expected": "'a'", + "got": "'a'", + "got_raw": "'a'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "petl-developers/petl", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "petl-developers/petl", + "expected": "second_row2", + "got": "second_row2", + "got_raw": "second_row2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "petl-developers/petl", + "expected": "('foofoo', 'barbar')", + "got": "('foofoo', 'barbar')", + "got_raw": "('foofoo', 'barbar')", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "petl-developers/petl", + "expected": "'n'", + "got": "'n'", + "got_raw": "'n'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet/pysen", + "expected": "BASE_DIR", + "got": "BASE_DIR", + "got_raw": "BASE_DIR", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet/pysen", + "expected": "test_components", + "got": "test_components", + "got_raw": "test_components", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet/pysen", + "expected": "GitRepositoryNotFoundError)", + "got": "git_utils.GitRepositoryNotFoundError)", + "got_raw": "git_utils.GitRepositoryNotFoundError)", + "exact_match": false, + "code_bleu": 0.47287080450158786, + "edit_similarity": 0.84375 + }, + { + "repo": "pfnet/pysen", + "expected": "logging_utils._PROCESS_STDERR", + "got": "logging_utils._get_process_output_level_name(logging.WARNING)", + "got_raw": "logging_utils._get_process_output_level_name(logging.WARNING)", + "exact_match": false, + "code_bleu": 0.21105340631872638, + "edit_similarity": 0.37777777777777777 + }, + { + "repo": "pfnet/pysen", + "expected": "{\"op1\": [\"node2\"], \"op3\": [\"node2\"]}", + "got": "{\"op1\": [\"node2\"], \"op2\": [], \"op3\": [\"node2\"]}", + "got_raw": "{\"op1\": [\"node2\"], \"op2\": [], \"op3\": [\"node2\"]}", + "exact_match": false, + "code_bleu": 0.6960884152052168, + "edit_similarity": 0.8674698795180723 + }, + { + "repo": "pfnet/pysen", + "expected": "settings_dir", + "got": "pathlib.Path(fname).parent", + "got_raw": "pathlib.Path(fname).parent", + "exact_match": false, + "code_bleu": 0.13485111859503685, + "edit_similarity": 0.21052631578947367 + }, + { + "repo": "pfnet/pysen", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pfnet/pysen", + "expected": "TypeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pfnet/pysen", + "expected": "4", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pfnet/pysen", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet/pysen", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet/pysen", + "expected": "{\"hoge.fuga\": mypy.MypySetting.strict()}", + "got": "{\"hoge.fuga\": mypy.MypySetting.strict()}", + "got_raw": "{\"hoge.fuga\": mypy.MypySetting.strict()}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet/pysen", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet/pysen", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet/pysen", + "expected": "\"plugin\")", + "got": "\"plugin\")", + "got_raw": "\"plugin\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet/pysen", + "expected": "\"%(message)s\"", + "got": "logging_utils._logging_output_fmt", + "got_raw": "logging_utils._logging_output_fmt", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.08695652173913043 + }, + { + "repo": "pfnet/pysen", + "expected": "{\"setup.cfg\"}", + "got": "{\"foo\", \"bar\"}", + "got_raw": "{\"foo\", \"bar\"}", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.37037037037037035 + }, + { + "repo": "pfnet/pysen", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pfnet/pysen", + "expected": "f\"{entry_point}_component1\"", + "got": "\"plugin\"", + "got_raw": "\"plugin\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "pfnet/pysen", + "expected": "6", + "got": "6", + "got_raw": "6", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pgjones/hypercorn", + "expected": "(\"127.0.0.2\", 0)", + "got": "(\"127.0.0.1\", 80)", + "got_raw": "(\"127.0.0.1\", 80)", + "exact_match": false, + "code_bleu": 0.32466791547509893, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "pgjones/hypercorn", + "expected": "trio.BrokenResourceError)", + "got": "ConnectionResetError)", + "got_raw": "ConnectionResetError)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "pgjones/hypercorn", + "expected": "[]", + "got": "[(\"sec-websocket-version\", \"13\")]", + "got_raw": "[(\"sec-websocket-version\", \"13\")]", + "exact_match": false, + "code_bleu": 0.20556680845025982, + "edit_similarity": 0.11428571428571428 + }, + { + "repo": "pgjones/hypercorn", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pgjones/hypercorn", + "expected": "[ wsproto.events.AcceptConnection( extra_headers=[ (b\"date\", b\"Thu, 01 Jan 1970 01:23:20 GMT\"), (b\"server\", b\"hypercorn-h11\"), ] ) ]", + "got": "[", + "got_raw": "[ # type: ignore\n wsproto.events.Event.headers(wsproto.Headers(headers=[(\"Sec-WebSocket-Accept\", \"dGhlIHNhbXBsZSBub25jZQ==\")]))", + "exact_match": false, + "code_bleu": 0.020503570948322777, + "edit_similarity": 0.015037593984962405 + }, + { + "repo": "pgjones/hypercorn", + "expected": "ASGIWebsocketState.CONNECTED", + "got": "ASGIWebsocketState.OPEN", + "got_raw": "ASGIWebsocketState.OPEN", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "pgjones/hypercorn", + "expected": "[", + "got": "[call(Response(stream_id=1, status_code=201, headers=[(b\"X-Special\", b\"Value\")]))]", + "got_raw": "[call(Response(stream_id=1, status_code=201, headers=[(b\"X-Special\", b\"Value\")]))]", + "exact_match": false, + "code_bleu": 0.044915755686574035, + "edit_similarity": 0.024096385542168676 + }, + { + "repo": "pgjones/hypercorn", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pgjones/hypercorn", + "expected": "\"http\"", + "got": "\"http\"", + "got_raw": "\"http\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pgjones/hypercorn", + "expected": "23", + "got": "23", + "got_raw": "23", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pgjones/hypercorn", + "expected": "[wsproto.events.CloseConnection(code=1000, reason=\"\")]", + "got": "[wsproto.events.CloseConnection(code=1000)]", + "got_raw": "[wsproto.events.CloseConnection(code=1000)]", + "exact_match": false, + "code_bleu": 0.6600838025950451, + "edit_similarity": 0.8865979381443299 + }, + { + "repo": "pgjones/hypercorn", + "expected": "socket.SO_TYPE)", + "got": "socket.SO_TYPE)", + "got_raw": "socket.SO_TYPE)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pgjones/hypercorn", + "expected": "[ {\"type\": \"http.response.start\", \"status\": 200, \"headers\": [(b\"content-length\", b\"13\")]}, {\"type\": \"http.response.body\", \"body\": b\"apix-/api/x/b\"}, {\"type\": \"http.response.start\", \"status\": 200, \"headers\": [(b\"content-length\", b\"10\")]}, {\"type\": \"http.response.body\", \"body\": b\"api-/api/b\"}, {\"type\": \"http.response.start\", \"status\": 404, \"headers\": [(b\"content-length\", b\"0\")]}, {\"type\": \"http.response.body\"}, ]", + "got": "[", + "got_raw": "[ # type: ignore\n {\"type\": \"http.response.start\", \"status\": 200, \"headers\": [(b\"content-length\", b\"10\")]}, # type: ignore\n {\"type\": \"http.response.body\", \"body\": b\"apix-x/b\".encode()}, # type: ignore\n {\"type\": \"http.response.start\", \"status\": 200, \"headers\": [(b\"content-length\", b\"10\")]}, # type: ignore\n {\"type\": \"http.response.body\", \"body\": b\"api-b\".encode", + "exact_match": false, + "code_bleu": 0.006193787057305838, + "edit_similarity": 0.004819277108433735 + }, + { + "repo": "pgjones/hypercorn", + "expected": "\"/\u4e2d\".encode().decode(\"latin-1\")", + "got": "\"/\u4e2d\"", + "got_raw": "\"/\u4e2d\"", + "exact_match": false, + "code_bleu": 0.05946035575013606, + "edit_similarity": 0.22857142857142856 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "\"uuid\")", + "got": "ticket.code)", + "got_raw": "ticket.code)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.3157894736842105 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "[1, 2, 3])", + "got": "[1, 2, 3])", + "got_raw": "[1, 2, 3])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "[TableB.table_a])", + "got": "[\"table_a\"])", + "got_raw": "[\"table_a\"])", + "exact_match": false, + "code_bleu": 0.3387551654364098, + "edit_similarity": 0.6896551724137931 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "\"Bob\")", + "got": "\"Bob\")", + "got_raw": "\"Bob\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "\"[]\")", + "got": "[])", + "got_raw": "[])", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.75 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "len(snapshot) == 2)", + "got": "snapshot)", + "got_raw": "snapshot)", + "exact_match": false, + "code_bleu": 0.1767766952966369, + "edit_similarity": 0.6428571428571429 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "list)", + "got": "list)", + "got_raw": "list)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "str)", + "got": "list[str])", + "got_raw": "list[str])", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.5714285714285714 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "\"l\")", + "got": "Shirt.Size.medium)", + "got_raw": "Shirt.Size.medium)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.09090909090909091 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "\"manager\")", + "got": "\"manager\")", + "got_raw": "\"manager\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "tables)", + "got": "tables)", + "got_raw": "tables)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "\"name\")", + "got": "Manager.name)", + "got_raw": "Manager.name)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.5 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "query.__str__())", + "got": "str(query))", + "got_raw": "str(query))", + "exact_match": false, + "code_bleu": 0.34020690871988585, + "edit_similarity": 0.5185185185185185 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "[TableA, TableB])", + "got": "sort_table_classes([TableA, TableB], [TableA, TableB])", + "got_raw": "\n sort_table_classes([TableA, TableB], [TableA, TableB])", + "exact_match": false, + "code_bleu": 0.4001601601922499, + "edit_similarity": 0.4788732394366197 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "\"Column\")", + "got": "Column)", + "got_raw": "Column)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.875 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "[test_date])", + "got": "[test_date])", + "got_raw": "[test_date])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "Integer().column_type)", + "got": "Band.manager.id)", + "got_raw": "Band.manager.id)", + "exact_match": false, + "code_bleu": 0.24446151121745047, + "edit_similarity": 0.3157894736842105 + }, + { + "repo": "piccolo-orm/piccolo", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "picoCTF/picoCTF", + "expected": "409", + "got": "400", + "got_raw": "400", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "picoCTF/picoCTF", + "expected": "422", + "got": "403", + "got_raw": "403", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "picoCTF/picoCTF", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "picoCTF/picoCTF", + "expected": "v", + "got": "v", + "got_raw": "v", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "picoCTF/picoCTF", + "expected": "403", + "got": "403", + "got_raw": "403", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "picoCTF/picoCTF", + "expected": "str(excep_info.value)", + "got": "str(excep_info)", + "got_raw": "str(excep_info)", + "exact_match": false, + "code_bleu": 0.4714045207910316, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "pimutils/khal", + "expected": "''", + "got": "'one\\n'", + "got_raw": "'one\\n'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "pimutils/khal", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pimutils/khal", + "expected": "'\\r\\n'.join([ 'BEGIN:VEVENT', 'SUMMARY:An Event', 'DTSTART;VALUE=DATE:20220702', 'DTEND;VALUE=DATE:20220703', 'DTSTAMP:20220702T140000Z', 'UID:E41JRQX2DB4P1AQZI86BAT7NHPBHPRIIHQKA', 'RRULE:FREQ=WEEKLY', 'END:VEVENT', '' ])", + "got": "textwrap.dedent(\"\"\"\\", + "got_raw": "textwrap.dedent(\"\"\"\\\n BEGIN:VEVENT\n DTSTAMP:20220702T140000Z\n UID:20220702T140000Z-1234567890@localhost\n DTSTART:20220702T000000Z\n DTEND:20220703T000000Z\n RRULE:FREQ=WEEKLY\n SUMMARY:An Event\n END:", + "exact_match": false, + "code_bleu": 0.059956985797992454, + "edit_similarity": 0.008264462809917356 + }, + { + "repo": "pimutils/khal", + "expected": "_get_TZIDs(part1)", + "got": "_get_TZIDs(part1)", + "got_raw": "_get_TZIDs(part1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pimutils/khal", + "expected": "InvalidSettingsError)", + "got": "InvalidSettingsError)", + "got_raw": "InvalidSettingsError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pimutils/khal", + "expected": "uid", + "got": "\"https://pimutils.org/ical/vdir/042MJ8B3SJNFXQOJL6P53OFMHJE8Z3VZWOU\"", + "got_raw": "\"https://pimutils.org/ical/vdir/042MJ8B3SJNFXQOJL6P53OFMHJE8Z3VZWOU\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.056338028169014086 + }, + { + "repo": "pimutils/khal", + "expected": "(5, 1)", + "got": "(5, 2)", + "got_raw": "(5, 2)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "pimutils/khal", + "expected": "vevents2[0]", + "got": "vevents2[0]", + "got_raw": "vevents2[0]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pimutils/khal", + "expected": "'Wednesday'", + "got": "'Wednesday'", + "got_raw": "'Wednesday'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pimutils/khal", + "expected": "8", + "got": "12", + "got_raw": "12", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pimutils/khal", + "expected": "comp_config[key]", + "got": "comp_config[key]", + "got_raw": "comp_config[key]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pimutils/khal", + "expected": "\"\"", + "got": "['John Doe']", + "got_raw": "['John Doe']", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "pimutils/khal", + "expected": "'2d'", + "got": "'2d'", + "got_raw": "'2d'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pimutils/khal", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pimutils/khal", + "expected": "{vdir[len(path):] for vdir in get_all_vdirs(path + '/**/')}", + "got": "{vdir[len(path):] for vdir in get_all_vdirs(path + '/**/*', recursive=True)}", + "got_raw": "{vdir[len(path):] for vdir in get_all_vdirs(path + '/**/*', recursive=True)}", + "exact_match": false, + "code_bleu": 0.7181417752544185, + "edit_similarity": 0.8740740740740741 + }, + { + "repo": "pimutils/khal", + "expected": "testList2", + "got": "testList2", + "got_raw": "testList2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pirxthepilot/wtfis", + "expected": "4", + "got": "5", + "got_raw": "5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pirxthepilot/wtfis", + "expected": "\"cloudflare.com\"", + "got": "\"1.1.1.1\"", + "got_raw": "\"1.1.1.1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.24 + }, + { + "repo": "pirxthepilot/wtfis", + "expected": "theme.heading_h1", + "got": "theme.urlhaus", + "got_raw": "theme.urlhaus", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.5517241379310345 + }, + { + "repo": "pirxthepilot/wtfis", + "expected": "HandlerException", + "got": "HandlerException", + "got_raw": "HandlerException", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pirxthepilot/wtfis", + "expected": "IpInfoMap.model_validate({})", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.20203050891044216, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "pirxthepilot/wtfis", + "expected": "\"IPWhois\"", + "got": "\"ipwhois\"", + "got_raw": "\"ipwhois\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pirxthepilot/wtfis", + "expected": "\"\"", + "got": "Text(\"\\n\")", + "got_raw": "Text(\"\\n\")", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "pirxthepilot/wtfis", + "expected": "[]", + "got": "[\"web\"]", + "got_raw": "[\"web\"]", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "piskvorky/smart_open", + "expected": "\"w\")", + "got": "\"w\")", + "got_raw": "\"w\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piskvorky/smart_open", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piskvorky/smart_open", + "expected": "limit", + "got": "limit", + "got_raw": "limit", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piskvorky/smart_open", + "expected": "0)", + "got": "2)", + "got_raw": "2)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "piskvorky/smart_open", + "expected": "3)", + "got": "100)", + "got_raw": "100)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "piskvorky/smart_open", + "expected": "CRIME_AND_PUNISHMENT[:len(as_text)]", + "got": "CAP_TEXT", + "got_raw": "CAP_TEXT", + "exact_match": false, + "code_bleu": 0.0625, + "edit_similarity": 0.18604651162790697 + }, + { + "repo": "piskvorky/smart_open", + "expected": "404", + "got": "404", + "got_raw": "404", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piskvorky/smart_open", + "expected": "data)", + "got": "data)", + "got_raw": "data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piskvorky/smart_open", + "expected": "text)", + "got": "text)", + "got_raw": "text)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piskvorky/smart_open", + "expected": "WebHdfsException)", + "got": "WebHdfsException)", + "got_raw": "WebHdfsException)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piskvorky/smart_open", + "expected": "from_smart_open", + "got": "from_smart_open", + "got_raw": "from_smart_open", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piskvorky/smart_open", + "expected": "b'')", + "got": "b'')", + "got_raw": "b'')", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "piskvorky/smart_open", + "expected": "text", + "got": "text", + "got_raw": "text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "AdSegmentPredictionList( ad_segments=[AdSegmentPrediction(segment_offset=12.0, confidence=0.86)], content_type=\"promotional_external\", confidence=0.91, )", + "got": "AdSegmentPredictionList(", + "got_raw": "AdSegmentPredictionList(\n ad_segments=[AdSegmentPrediction(segment_offset=12.0, confidence=0.86)], content_type=\"promotional_external\", confidence=0.91\n )", + "exact_match": false, + "code_bleu": 0.054392829322042126, + "edit_similarity": 0.2711864406779661 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "\"client_error\"", + "got": "\"client_error\"", + "got_raw": "\"client_error\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "\"rate_limit\"", + "got": "\"rate_limit\"", + "got_raw": "\"rate_limit\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "404", + "got": "404", + "got_raw": "404", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "50000", + "got": "50000", + "got_raw": "50000", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "\"https://example.com/podcast.mp3\"", + "got": "\"https://example.com/podcast.mp3\"", + "got_raw": "\"https://example.com/podcast.mp3\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "\"Authentication required.\"", + "got": "\"missing user\"", + "got_raw": "\"missing user\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "AdSegmentPredictionList( ad_segments=[AdSegmentPrediction(segment_offset=12.0, confidence=0.86)], content_type=\"promotional_external\", confidence=0.92, )", + "got": "AdSegmentPredictionList(", + "got_raw": "AdSegmentPredictionList(\n ad_segments=[AdSegmentPrediction(segment_offset=12.0, confidence=0.86)], content_type=\"promotional_external\", confidence=0.92\n )", + "exact_match": false, + "code_bleu": 0.054392829322042126, + "edit_similarity": 0.2711864406779661 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "202", + "got": "200", + "got_raw": "200", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "mock_feed", + "got": "mock_feed", + "got_raw": "mock_feed", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "\"Test Episode\"", + "got": "\"Test Episode\"", + "got_raw": "\"Test Episode\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "30.0", + "got": "30000", + "got_raw": "30000", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "[ mock.call(\"whitelist_post\", {\"post_id\": post_id}, wait=True), mock.call(\"increment_download_count\", {\"post_id\": post_id}, wait=False), ])", + "got": "writer_action_args = mock_writer.action.call_args[0]", + "got_raw": "writer_action_args = mock_writer.action.call_args[0]", + "exact_match": false, + "code_bleu": 0.044838883822215445, + "edit_similarity": 0.15706806282722513 + }, + { + "repo": "podly-pure-podcasts/podly_pure_podcasts", + "expected": "[0, 1, 2, 3, 4, 5]", + "got": "[2, 3, 4, 5]", + "got_raw": "[2, 3, 4, 5]", + "exact_match": false, + "code_bleu": 0.6255706178837387, + "edit_similarity": 0.8 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "'system'", + "got": "'/path/to/exe'", + "got_raw": "'/path/to/exe'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "(('cmd',),)", + "got": "[('cmd',)]", + "got_raw": "[('cmd',)]", + "exact_match": false, + "code_bleu": 0.5269612739582266, + "edit_similarity": 0.7619047619047619 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "'~'", + "got": "'~'", + "got_raw": "'~'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "'bye\\n'", + "got": "'bye\\n'", + "got_raw": "'bye\\n'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "'repo'", + "got": "'repo'", + "got_raw": "'repo'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "f2.read()", + "got": "f2.read()", + "got_raw": "f2.read()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "stdout", + "got": "stdout", + "got_raw": "stdout", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "out", + "got": "out", + "got_raw": "out", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "'cd'", + "got": "'hazmat'", + "got_raw": "'hazmat'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "'python3.12'", + "got": "'/fake/path/found/python3.12'", + "got_raw": "'/fake/path/found/python3.12'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5853658536585366 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "'system'", + "got": "'/path/to/exe'", + "got_raw": "'/path/to/exe'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "f'{subdir}\\na\\nb\\n'", + "got": "'./subdir\\na\\nb\\n'", + "got_raw": "'./subdir\\na\\nb\\n'", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.8648648648648649 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "''", + "got": "''", + "got_raw": "''", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "'f5\\nf6\\n'", + "got": "'f4\\n'", + "got_raw": "'f4\\n'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.625 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "('echo', 'hi')", + "got": "['echo', 'hi']", + "got_raw": "['echo', 'hi']", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "out.decode())", + "got": "out)", + "got_raw": "out)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "pre-commit/pre-commit", + "expected": "()", + "got": "'root'", + "got_raw": "'root'", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "prkumar/uplink", + "expected": "method", + "got": "method", + "got_raw": "method", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "prkumar/uplink", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "prkumar/uplink", + "expected": "\"example.com\"", + "got": "\"http://example.com\"", + "got_raw": "\"http://example.com\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7878787878787878 + }, + { + "repo": "prkumar/uplink", + "expected": "obj)", + "got": "obj)", + "got_raw": "obj)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "prkumar/uplink", + "expected": "data", + "got": "data", + "got_raw": "data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "prkumar/uplink", + "expected": "{\"owner\": \"prkumar\", \"name\": \"uplink\"}", + "got": "Repo(owner=User(id=1, name=\"prkumar\"), name=\"uplink\")", + "got_raw": "Repo(owner=User(id=1, name=\"prkumar\"), name=\"uplink\")", + "exact_match": false, + "code_bleu": 0.07658412276041004, + "edit_similarity": 0.6373626373626373 + }, + { + "repo": "prkumar/uplink", + "expected": "SIMPLE_RESPONSE", + "got": "SIMPLE_RESPONSE", + "got_raw": "SIMPLE_RESPONSE", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "prkumar/uplink", + "expected": "exceptions.SSLError)", + "got": "exceptions.HTTPError)", + "got_raw": "exceptions.HTTPError)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8292682926829268 + }, + { + "repo": "prkumar/uplink", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "prkumar/uplink", + "expected": "converter_factory_mock", + "got": "converter_factory_mock", + "got_raw": "converter_factory_mock", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "prkumar/uplink", + "expected": "uri", + "got": "uri", + "got_raw": "uri", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "prkumar/uplink", + "expected": "basic_auth", + "got": "basic_auth", + "got_raw": "basic_auth", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "prkumar/uplink", + "expected": "commands.MissingArgumentAnnotations)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.2127659574468085 + }, + { + "repo": "prkumar/uplink", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "prkumar/uplink", + "expected": "expected_error", + "got": "expected_error", + "got_raw": "expected_error", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "prkumar/uplink", + "expected": "\"/graphql2\"", + "got": "\"/graphql\"", + "got_raw": "\"/graphql\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9523809523809523 + }, + { + "repo": "prkumar/uplink", + "expected": "auth_str", + "got": "auth_str", + "got_raw": "auth_str", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "prkumar/uplink", + "expected": "\"2\"", + "got": "\"2\"", + "got_raw": "\"2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "prkumar/uplink", + "expected": "http_client_mock", + "got": "http_client_mock", + "got_raw": "http_client_mock", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "prkumar/uplink", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "protectai/llm-guard", + "expected": "expected_score", + "got": "expected_score", + "got_raw": "expected_score", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "protectai/llm-guard", + "expected": "expected_valid", + "got": "expected_valid", + "got_raw": "expected_valid", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "protectai/llm-guard", + "expected": "output", + "got": "output", + "got_raw": "output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "protectai/llm-guard", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "protectai/llm-guard", + "expected": "expected_output", + "got": "expected_output", + "got_raw": "expected_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "protectai/llm-guard", + "expected": "expected_score", + "got": "expected_score", + "got_raw": "expected_score", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "protectai/llm-guard", + "expected": "expected_valid", + "got": "expected_valid", + "got_raw": "expected_valid", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "protectai/llm-guard", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "protectai/llm-guard", + "expected": "expected_valid", + "got": "expected_valid", + "got_raw": "expected_valid", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "protectai/llm-guard", + "expected": "expected_score", + "got": "expected_score", + "got_raw": "expected_score", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "protectai/llm-guard", + "expected": "expected_prompt", + "got": "expected_prompt", + "got_raw": "expected_prompt", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "protectai/llm-guard", + "expected": "expected_valid", + "got": "expected_valid", + "got_raw": "expected_valid", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "protectai/llm-guard", + "expected": "expected_valid", + "got": "expected_valid", + "got_raw": "expected_valid", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "protectai/llm-guard", + "expected": "expected_score", + "got": "expected_score", + "got_raw": "expected_score", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "protectai/llm-guard", + "expected": "expected_score", + "got": "expected_score", + "got_raw": "expected_score", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "py2many/py2many", + "expected": "12", + "got": "12", + "got_raw": "12", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "py2many/py2many", + "expected": "parse( \"decltype(true) x;\", \"if(10) {\", \"x = true;\", \"} else {\", \"x = false;\", \"}\", \"auto y = x;\", )", + "got": "\"bool x;\\nbool y;\\nif (10) {\\n x = true;\\n} else {\\n x = false;\\n}\\ny = x;\\n\"", + "got_raw": "\"bool x;\\nbool y;\\nif (10) {\\n x = true;\\n} else {\\n x = false;\\n}\\ny = x;\\n\"", + "exact_match": false, + "code_bleu": 0.029411764705882353, + "edit_similarity": 0.5698324022346368 + }, + { + "repo": "py2many/py2many", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "py2many/py2many", + "expected": "\"3\"", + "got": "\"int\"", + "got_raw": "\"int\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "py2many/py2many", + "expected": "{dunder_init_dot_py}", + "got": "{dunder_init_dot_py}", + "got_raw": "{dunder_init_dot_py}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "py2many/py2many", + "expected": "6 + 8j", + "got": "6 + 8j", + "got_raw": "6 + 8j", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "py2many/py2many", + "expected": "5", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "py2many/py2many", + "expected": "textwrap.dedent(expected)", + "got": "expected", + "got_raw": "expected", + "exact_match": false, + "code_bleu": 0.09910059291689341, + "edit_similarity": 0.48484848484848486 + }, + { + "repo": "py2many/py2many", + "expected": "\"2\"", + "got": "\"list\"", + "got_raw": "\"list\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "py2many/py2many", + "expected": "failure_count", + "got": "failure_count", + "got_raw": "failure_count", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "py2many/py2many", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "py2many/py2many", + "expected": "\"std::vector values = {0, 1, 2, 3};\"", + "got": "parse(\"int values[] = {0, 1, 2, 3};\")", + "got_raw": "parse(\"int values[] = {0, 1, 2, 3};\")", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.627906976744186 + }, + { + "repo": "py2many/py2many", + "expected": "Path(\"foo.rs\")", + "got": "Path(\"foo.rs\")", + "got_raw": "Path(\"foo.rs\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "g(2,1)", + "got": "g(2,1)", + "got_raw": "g(2,1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "f5()", + "got": "f5()", + "got_raw": "f5()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "f1(3)", + "got": "f1(3)", + "got_raw": "f1(3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "f3(x)", + "got": "f3(x)", + "got_raw": "f3(x)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "f2(4)", + "got": "f2(4)", + "got_raw": "f2(4)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "atts", + "got": "atts", + "got_raw": "atts", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "p_func()", + "got": "p_func()", + "got_raw": "p_func()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "swp(x,y,z)", + "got": "swp(x,y,z)", + "got_raw": "swp(x,y,z)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "x_pyt", + "got": "x_pyt", + "got_raw": "x_pyt", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "epyc_f(1)", + "got": "epyc_f(1)", + "got_raw": "epyc_f(1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "f2(a)", + "got": "f2(a)", + "got_raw": "f2(a)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "f2( 2 )", + "got": "f2( 2 )", + "got_raw": "f2( 2 )", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "modnew.x2", + "got": "modnew.x2", + "got_raw": "modnew.x2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "f14(6)", + "got": "f14(6)", + "got_raw": "f14(6)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "p_lose()", + "got": "p_lose()", + "got_raw": "p_lose()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "f9(4)", + "got": "f9(4)", + "got_raw": "f9(4)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "f3(4.5)", + "got": "f3(4.5)", + "got_raw": "f3(4.5)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "f2()", + "got": "f2()", + "got_raw": "f2()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "f1(None)", + "got": "f1(None)", + "got_raw": "f1(None)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyccel/pyccel", + "expected": "f()", + "got": "f()", + "got_raw": "f()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "dummy_md5", + "got": "dummy_md5", + "got_raw": "dummy_md5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "59", + "got": "59", + "got_raw": "59", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "dict_b", + "got": "dict_b", + "got_raw": "dict_b", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "Plugins)", + "got": "Plugins)", + "got_raw": "Plugins)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "job_name", + "got": "job_name", + "got_raw": "job_name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "{ \"foo\": \"http://halob:8080/job/foo/\", \"test_jenkinsapi\": \"http://halob:8080/job/test_jenkinsapi/\", }", + "got": "DATA[\"jobs\"]", + "got_raw": "DATA[\"jobs\"]", + "exact_match": false, + "code_bleu": 0.12085501589427072, + "edit_similarity": 0.10619469026548672 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "\"http://localhost:8080/job/Folder1/api/python\")", + "got": "()", + "got_raw": "()", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.04081632653061224 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "id(jenkins)", + "got": "id(jenkins)", + "got_raw": "id(jenkins)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "\"origin/unstable\"", + "got": "\"master\"", + "got_raw": "\"master\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "jenkins", + "got": "jenkins", + "got_raw": "jenkins", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "AttributeError)", + "got": "AttributeError)", + "got_raw": "AttributeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "DATA_JOBS", + "got": "[\"test_job1\", \"test_job2\", \"test_job3\", \"test_job4\"]", + "got_raw": "[\"test_job1\", \"test_job2\", \"test_job3\", \"test_job4\"]", + "exact_match": false, + "code_bleu": 0.11868405219520975, + "edit_similarity": 0.03278688524590164 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "\"[]\")", + "got": "\"Plugins: 0\")", + "got_raw": "\"Plugins: 0\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "creds", + "got": "creds", + "got_raw": "creds", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "node_labels", + "got": "node_labels", + "got_raw": "node_labels", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "\"test1\"", + "got": "\"test1\"", + "got_raw": "\"test1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pycontribs/jenkinsapi", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "'abc'", + "got": "'abc'", + "got_raw": "'abc'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "[1, 3]", + "got": "[1, 3]", + "got_raw": "[1, 3]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "v", + "got": "v", + "got_raw": "v", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "Any", + "got": "Any", + "got_raw": "Any", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "{1: 1}", + "got": "{'1': 1}", + "got_raw": "{'1': 1}", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "msg", + "got": "msg", + "got_raw": "msg", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "float('inf')", + "got": "1.0", + "got_raw": "1.0", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.0 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "{}", + "got": "{}", + "got_raw": "{}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "v", + "got": "v", + "got_raw": "v", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "8", + "got": "8", + "got_raw": "8", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "repr(s)", + "got": "str(s)", + "got_raw": "str(s)", + "exact_match": false, + "code_bleu": 0.668740304976422, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "'baz'", + "got": "'baz'", + "got_raw": "'baz'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "m", + "got": "m", + "got_raw": "m", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-core", + "expected": "b'1'", + "got": "'1'", + "got_raw": "'1'", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "pydantic/pydantic-settings", + "expected": "error_type, match=msg_fragment)", + "got": "error_type)", + "got_raw": "error_type)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5238095238095238 + }, + { + "repo": "pydantic/pydantic-settings", + "expected": "'bar'", + "got": "'bar'", + "got_raw": "'bar'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-settings", + "expected": "'test-value'", + "got": "'test-value'", + "got_raw": "'test-value'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-settings", + "expected": "'world!'", + "got": "'world!'", + "got_raw": "'world!'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-settings", + "expected": "'foo'", + "got": "'foo'", + "got_raw": "'foo'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-settings", + "expected": "'world!'", + "got": "'world!'", + "got_raw": "'world!'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-settings", + "expected": "A()", + "got": "A()", + "got_raw": "A()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-settings", + "expected": "'default'", + "got": "'default'", + "got_raw": "'default'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-settings", + "expected": "123", + "got": "123", + "got_raw": "123", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pydantic/pydantic-settings", + "expected": "'JsonConfigSettingsSource(json_file=config.json)'", + "got": "f\"JsonConfigSettingsSource(BaseSettings, Path('config.json'))\"", + "got_raw": "f\"JsonConfigSettingsSource(BaseSettings, Path('config.json'))\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.7027027027027027 + }, + { + "repo": "pydantic/pydantic-settings", + "expected": "{'x': 1, 'y': 3}", + "got": "{'x': 1, 'y': 3}", + "got_raw": "{'x': 1, 'y': 3}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "3", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "2", + "got": "case", + "got_raw": "case", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "inst.nplc", + "got": "inst.nplc", + "got_raw": "inst.nplc", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "t", + "got": "t", + "got_raw": "t", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "inst.output", + "got": "inst.output", + "got_raw": "inst.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "str", + "got": "float", + "got_raw": "float", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.25 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "[5]", + "got": "5", + "got_raw": "5", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "\"a1\"", + "got": "\"a1\"", + "got_raw": "\"a1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "f", + "got": "f", + "got_raw": "f", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "p", + "got": "p", + "got_raw": "p", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pymeasure/pymeasure", + "expected": "7", + "got": "\"7\"", + "got_raw": "\"7\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "pypa/setuptools-scm", + "expected": "True", + "got": "False", + "got_raw": "False", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "pypa/setuptools-scm", + "expected": "no_guess_dev_version(version)", + "got": "\"1.0\"", + "got_raw": "\"1.0\"", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.0 + }, + { + "repo": "pypa/setuptools-scm", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/setuptools-scm", + "expected": "tag_regex", + "got": "tag_regex", + "got_raw": "tag_regex", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/setuptools-scm", + "expected": "_sep({\"adir/space file\", \"adir/filea\"})", + "got": "_sep({\"adir/space file\"})", + "got_raw": "_sep({\"adir/space file\"})", + "exact_match": false, + "code_bleu": 0.5303300858899107, + "edit_similarity": 0.78125 + }, + { + "repo": "pypa/setuptools-scm", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/setuptools-scm", + "expected": "logging.DEBUG", + "got": "logging.DEBUG", + "got_raw": "logging.DEBUG", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/setuptools-scm", + "expected": "set()", + "got": "set()", + "got_raw": "set()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/setuptools-scm", + "expected": "result", + "got": "expected", + "got_raw": "expected", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "pypa/setuptools-scm", + "expected": "\"C:/path/to/file\"", + "got": "\"C:\\\\path\\\\to\\\\file\"", + "got_raw": "\"C:\\\\path\\\\to\\\\file\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7567567567567568 + }, + { + "repo": "pypa/setuptools-scm", + "expected": "_sep({\"adir/filea\"})", + "got": "_sep({\"adir/filea\"})", + "got_raw": "_sep({\"adir/filea\"})", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/setuptools-scm", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/setuptools-scm", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pypa/setuptools-scm", + "expected": "LookupError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "pyparsing/pyparsing", + "expected": "exc_type, expected_msg)", + "got": "exc_type, expected_msg)", + "got_raw": "exc_type, expected_msg)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyparsing/pyparsing", + "expected": "9", + "got": "9", + "got_raw": "9", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyparsing/pyparsing", + "expected": "diag_html_capture.getvalue())", + "got": "diag_html_capture.getvalue())", + "got_raw": "diag_html_capture.getvalue())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyparsing/pyparsing", + "expected": "1234", + "got": "1234", + "got_raw": "1234", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyparsing/pyparsing", + "expected": "exc_type)", + "got": "exc_type, msg=msg)", + "got_raw": "exc_type, msg=msg)", + "exact_match": false, + "code_bleu": 0.24446151121745047, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "pyparsing/pyparsing", + "expected": "e", + "got": "e", + "got_raw": "e", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyparsing/pyparsing", + "expected": "\"read_stmt\"", + "got": "\"read\"", + "got_raw": "\"read\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "pyparsing/pyparsing", + "expected": "\"123\"", + "got": "123", + "got_raw": "123", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "pyparsing/pyparsing", + "expected": "ParseException)", + "got": "ParseException)", + "got_raw": "ParseException)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyranges/pyranges", + "expected": "bedtools_df.sort_values(\"Chromosome Start Strand\".split()))", + "got": "natsorted(bedtools_df.sort_values(by=[\"Chromosome\", \"Start\", \"End\"]).reset_index(drop=True)))", + "got_raw": "natsorted(bedtools_df.sort_values(by=[\"Chromosome\", \"Start\", \"End\"]).reset_index(drop=True)))", + "exact_match": false, + "code_bleu": 0.1912481314888403, + "edit_similarity": 0.6842105263157895 + }, + { + "repo": "pyranges/pyranges", + "expected": "lc", + "got": "lc", + "got_raw": "lc", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyranges/pyranges", + "expected": "bedtools_df)", + "got": "bedtools_df)", + "got_raw": "bedtools_df)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pyranges/pyranges", + "expected": "len(_introns)", + "got": "len(genes)", + "got_raw": "len(genes)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6086956521739131 + }, + { + "repo": "pyranges/pyranges", + "expected": "bedtools_df)", + "got": "bedtools_df)", + "got_raw": "bedtools_df)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytest-dev/pytest-bdd", + "expected": "5", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pytest-dev/pytest-bdd", + "expected": "\"re\"", + "got": "\"re\"", + "got_raw": "\"re\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytest-dev/pytest-bdd", + "expected": "[ 12, 5.0, \"7\", 5, 4.0, \"1\", 3, 9.0, \"-6\", 1, 4.0, \"-3\", ]", + "got": "[", + "got_raw": "[\n 12, 5, 7, 12, 4, 1, 3, 9, 3, 9, -6, 1, 4, -3, 1, 4, -3, 5, 4, 1, 5, 4, 1, 1, 4, 1, 1, 1, 4, 1, 1, 1, 4, 1, 1, 1, 4, 1, 1, 1, 4", + "exact_match": false, + "code_bleu": 0.022869367596206176, + "edit_similarity": 0.03389830508474576 + }, + { + "repo": "pytest-dev/pytest-bdd", + "expected": "42", + "got": "42", + "got_raw": "42", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytest-dev/pytest-bdd", + "expected": "2", + "got": "5", + "got_raw": "5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pytest-dev/pytest-bdd", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytest-dev/pytest-xdist", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytest-dev/pytest-xdist", + "expected": "8", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pytest-dev/pytest-xdist", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "pytest-dev/pytest-xdist", + "expected": "self.node2pending", + "got": "self.node2collection", + "got_raw": "self.node2collection", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7027027027027027 + }, + { + "repo": "pytest-dev/pytest-xdist", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytest-dev/pytest-xdist", + "expected": "self.node2pending", + "got": "self.node2collection", + "got_raw": "self.node2collection", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7027027027027027 + }, + { + "repo": "python-attrs/attrs", + "expected": "c.a1", + "got": "c.a1", + "got_raw": "c.a1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "i2", + "got": "i2", + "got_raw": "i2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "result", + "got": "result == { \"a\": {\"x\": \"2020-07-01T00:00:00\", \"y\": [\"2020-07-01T00:00:00\"]}, \"b\": [{\"x\": \"2020-07-02T00:00:00\", \"y\": [\"2020-07-02T00:00:00\"]}], \"c\": {\"spam\": {\"x\": \"2020-07-03T", + "got_raw": "result == { \"a\": {\"x\": \"2020-07-01T00:00:00\", \"y\": [\"2020-07-01T00:00:00\"]}, \"b\": [{\"x\": \"2020-07-02T00:00:00\", \"y\": [\"2020-07-02T00:00:00\"]}], \"c\": {\"spam\": {\"x\": \"2020-07-03T", + "exact_match": true, + "code_bleu": 0.02452625300522044, + "edit_similarity": 0.06593406593406594 + }, + { + "repo": "python-attrs/attrs", + "expected": "src", + "got": "src", + "got_raw": "src", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "ei.value.x", + "got": "ei.value.x", + "got_raw": "ei.value.x", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "simple_class()", + "got": "simple_class()", + "got_raw": "simple_class()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "pf.x", + "got": "pf.x", + "got_raw": "pf.x", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "C", + "got": "C", + "got_raw": "C", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "E)", + "got": "E)", + "got_raw": "E)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "fields_type.__name__", + "got": "fields_type.__name__", + "got_raw": "fields_type.__name__", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "C(1, 2, 3)", + "got": "C(1, 2, 3)", + "got_raw": "C(1, 2, 3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "tuple(subs.values())", + "got": "subs.values()", + "got_raw": "subs.values()", + "exact_match": false, + "code_bleu": 0.625, + "edit_similarity": 0.7878787878787878 + }, + { + "repo": "python-attrs/attrs", + "expected": "c(None)", + "got": "c(None)", + "got_raw": "c(None)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "recwarn.list", + "got": "recwarn.list", + "got_raw": "recwarn.list", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-attrs/attrs", + "expected": "AttributeError, match=\"no attribute 'pop'\")", + "got": "TypeError, match=\"not support item assignment\")", + "got_raw": "TypeError, match=\"not support item assignment\")", + "exact_match": false, + "code_bleu": 0.4347208719449915, + "edit_similarity": 0.5111111111111111 + }, + { + "repo": "python-escpos/python-escpos", + "expected": "b\"\\x1d(L\\x0b\\x000p0\\x01\\x011\\x01\\x00\\x01\\x00\\x00\\x1d(L\\x02\\x0002\"", + "got": "instance._buffer", + "got_raw": "instance._buffer", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.024691358024691357 + }, + { + "repo": "python-escpos/python-escpos", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-escpos/python-escpos", + "expected": "devicenotfounderror)", + "got": "devicenotfounderror)", + "got_raw": "devicenotfounderror)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-escpos/python-escpos", + "expected": "b\"\\x1bt\\x00? ist teuro.\"", + "got": "b\"\\x1bt\\x00\u20ac ist teuro.\\x00\"", + "got_raw": "b\"\\x1bt\\x00\u20ac ist teuro.\\x00\"", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8846153846153846 + }, + { + "repo": "python-escpos/python-escpos", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-escpos/python-escpos", + "expected": "caplog.text", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-escpos/python-escpos", + "expected": "str(w[-1].message)", + "got": "str(w[-1].message)", + "got_raw": "str(w[-1].message)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-escpos/python-escpos", + "expected": "AttributeError)", + "got": "escpos.exceptions.ESCPOSException)", + "got_raw": "escpos.exceptions.ESCPOSException)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.12244897959183673 + }, + { + "repo": "python-escpos/python-escpos", + "expected": "ImageWidthError)", + "got": "ImageWidthError)", + "got_raw": "ImageWidthError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-escpos/python-escpos", + "expected": "b\"_ ist teuro.\"", + "got": "b\"\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40", + "got_raw": "b\"\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40\\x1b\\x40", + "exact_match": false, + "code_bleu": 0.01391004452078366, + "edit_similarity": 0.021621621621621623 + }, + { + "repo": "python-escpos/python-escpos", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-escpos/python-escpos", + "expected": "lower_part.width", + "got": "100", + "got_raw": "100", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.0 + }, + { + "repo": "python-escpos/python-escpos", + "expected": "b\"\\x1dv0\\x00\\x01\\x00\\x01\\x00\\x00\"", + "got": "b\"\\x1b\\x33\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00", + "got_raw": "b\"\\x1b\\x33\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00", + "exact_match": false, + "code_bleu": 0.01391004452078366, + "edit_similarity": 0.1477832512315271 + }, + { + "repo": "python-escpos/python-escpos", + "expected": "width_expected", + "got": "width_expected", + "got_raw": "width_expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-jsonschema/jsonschema", + "expected": "\"anyOf\")", + "got": "\"type\")", + "got_raw": "\"type\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "python-jsonschema/jsonschema", + "expected": "FormatError)", + "got": "BOOM)", + "got_raw": "BOOM)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.11764705882352941 + }, + { + "repo": "python-jsonschema/jsonschema", + "expected": "tree)", + "got": "tree)", + "got_raw": "tree)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-jsonschema/jsonschema", + "expected": "\"oneOf\")", + "got": "\"type\")", + "got_raw": "\"type\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "python-jsonschema/jsonschema", + "expected": "\"type\")", + "got": "\"type\")", + "got_raw": "\"type\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-jsonschema/jsonschema", + "expected": "ValidationError)", + "got": "ValidationError)", + "got_raw": "ValidationError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-jsonschema/jsonschema", + "expected": "stderr)", + "got": "stderr)", + "got_raw": "stderr)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "[ test_cfg.strpath ]", + "got": "[test_cfg.strpath]", + "got_raw": "[test_cfg.strpath]", + "exact_match": false, + "code_bleu": 1.0, + "edit_similarity": 0.9473684210526315 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "\"isabs\"", + "got": "\"i\"", + "got_raw": "\"i\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "expected_call_args)", + "got": "expected_call_args)", + "got_raw": "expected_call_args)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "\"TEXT\"", + "got": "\"TEXT\"", + "got_raw": "\"TEXT\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "\"a = 123\\n\\n\\ndef func():\\n pass\\n\"", + "got": "DOC", + "got_raw": "DOC", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "[ {\"range\": edit_range, \"newText\": \"import pathlib\\n\"} ]", + "got": "[edit_range]", + "got_raw": "[edit_range]", + "exact_match": false, + "code_bleu": 0.14653226343048298, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "pylsp_hover(doc._config, doc, numpy_sin_hov_position)[\"contents\"][ \"value\" ]", + "got": "contents in pylsp_hover(doc._config, doc, numpy_sin_hov_position)[", + "got_raw": "contents in pylsp_hover(doc._config, doc, numpy_sin_hov_position)[\n \"contents\"\n ][\"value\"]", + "exact_match": false, + "code_bleu": 0.6813435516124459, + "edit_similarity": 0.7605633802816901 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "lsp.DiagnosticSeverity.Warning", + "got": "lsp.DiagnosticSeverity.Warning", + "got_raw": "lsp.DiagnosticSeverity.Warning", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "{ \"code\": \"D100\", \"message\": \"D100: Missing docstring in public module\", \"severity\": lsp.DiagnosticSeverity.Warning, \"range\": { \"start\": {\"line\": 0, \"character\": 0}, \"end\": {\"line\": 0, \"character\": 11}, }, \"source\": \"pydocstyle\", }", + "got": "lsp.Diagnostic(", + "got_raw": "lsp.Diagnostic( # noqa: E203\n range=lsp.Range( # noqa: E203\n start=lsp.Position(line=0, character=0), end=lsp.Position(line=0, character=1)\n ),\n message=\"Missing module docstring\",\n severity=lsp.DiagnosticSeverity.Warning,\n code=\"D100\",\n codeDescription=None,\n source=\"pydocstyle\",\n )", + "exact_match": false, + "code_bleu": 0.037087906235418654, + "edit_similarity": 0.11382113821138211 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "{\"line\": 5, \"character\": 1}", + "got": "{\"line\": 3, \"character\": 10}", + "got_raw": "{\"line\": 3, \"character\": 10}", + "exact_match": false, + "code_bleu": 0.392814650900513, + "edit_similarity": 0.9454545454545454 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "\"parameter docstring\"", + "got": "\"parameter docstring\"", + "got_raw": "\"parameter docstring\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "\"foo()\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "pylsp_hover(doc._config, doc, numpy_hov_position_3)[\"contents\"][\"value\"]", + "got": "contents in pylsp_hover(doc._config, doc, numpy_sin_hov_position)[\"contents\"]", + "got_raw": "contents in pylsp_hover(doc._config, doc, numpy_sin_hov_position)[\"contents\"]", + "exact_match": false, + "code_bleu": 0.6317856621154437, + "edit_similarity": 0.8187919463087249 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "\"0Hello12345678901234567890123456789\"", + "got": "\"Hello\"", + "got_raw": "\"Hello\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3181818181818182 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-lsp/python-lsp-server", + "expected": "\"import sys\\n\"", + "got": "\"line 1\"", + "got_raw": "\"line 1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.36363636363636365 + }, + { + "repo": "python-poetry/cleo", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/cleo", + "expected": "\"black\"", + "got": "\"black\"", + "got_raw": "\"black\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/cleo", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/cleo", + "expected": "\"Foo description\"", + "got": "\"Foo description\"", + "got_raw": "\"Foo description\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/cleo", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/cleo", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/cleo", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/cleo", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/cleo", + "expected": "\"Parent Command.\"", + "got": "\"Parent command\"", + "got_raw": "\"Parent command\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "python-poetry/cleo", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/cleo", + "expected": "[\"bar\"]", + "got": "[\"bar\"]", + "got_raw": "[\"bar\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/cleo", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/cleo", + "expected": "\"Bar\"", + "got": "\"bar\"", + "got_raw": "\"bar\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"Django\"", + "got": "\"Django\"", + "got_raw": "\"Django\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "3", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "[with_includes / \"bar/baz.py\"]", + "got": "[\"bar\"]", + "got_raw": "[\"bar\"]", + "exact_match": false, + "code_bleu": 0.29999999999999993, + "edit_similarity": 0.3783783783783784 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"MIT\"", + "got": "\"MIT\"", + "got_raw": "\"MIT\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "release1", + "got": "release1", + "got_raw": "release1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "open", + "got": "open", + "got_raw": "open", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "parsed", + "got": "parsed", + "got_raw": "parsed", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "Path(path)", + "got": "Path(path)", + "got_raw": "Path(path)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "spec2", + "got": "spec2", + "got_raw": "spec2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"*\"", + "got": "\"~1.0\"", + "got_raw": "\"~1.0\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"demo-1.0.0.whl\"", + "got": "link.url", + "got_raw": "link.url", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.08333333333333333 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"2.18.0\"", + "got": "\"==2.18.0\"", + "got_raw": "\"==2.18.0\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "v124", + "got": "VersionRange(v124, v200)", + "got_raw": "VersionRange(v124, v200)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "{\"sha256\": file_checksum}", + "got": "{\"sha256\": file_checksum}", + "got_raw": "{\"sha256\": file_checksum}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "\"WARNING\"", + "got": "\"ERROR\"", + "got_raw": "\"ERROR\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.375 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "name", + "got": "name", + "got_raw": "name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "python-poetry/poetry-core", + "expected": "parsed.name", + "got": "parsed.name", + "got_raw": "parsed.name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "trie", + "got": "trie", + "got_raw": "trie", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "wo_trie", + "got": "wo_trie", + "got_raw": "wo_trie", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "pytries/marisa-trie", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "i)", + "got": "i)", + "got_raw": "i)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"dd-a\"", + "got": "\"dd-a\"", + "got_raw": "\"dd-a\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"a\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "qtile/qtile", + "expected": "\"25\"", + "got": "\"10 Canto, 9 Slashdot, 6 MintyFresh\"", + "got_raw": "\"10 Canto, 9 Slashdot, 6 MintyFresh\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.1 + }, + { + "repo": "qtile/qtile", + "expected": "[\"1\"]", + "got": "[\"1\"]", + "got_raw": "[\"1\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"|\"", + "got": "\"|\"", + "got_raw": "\"|\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"max\"", + "got": "\"screen_split\"", + "got_raw": "\"screen_split\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.21052631578947367 + }, + { + "repo": "qtile/qtile", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"b\"", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "qtile/qtile", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "RED", + "got": "RED", + "got_raw": "RED", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "\"a\"", + "got": "\"a\"", + "got_raw": "\"a\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qtile/qtile", + "expected": "s", + "got": "s", + "got_raw": "s", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "custom_pid", + "got": "custom_pid", + "got_raw": "custom_pid", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "\"My Activity\"", + "got": "\"My Activity\"", + "got_raw": "\"My Activity\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "4000", + "got": "4000", + "got_raw": "4000", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "mock_response", + "got": "mock_response", + "got_raw": "mock_response", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "qwertyquerty/pypresence", + "expected": "client_id", + "got": "client_id", + "got_raw": "client_id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ramnes/notion-sdk-py", + "expected": "{}", + "got": "mock_response", + "got_raw": "mock_response", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "ramnes/notion-sdk-py", + "expected": "503", + "got": "503", + "got_raw": "503", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ramnes/notion-sdk-py", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ramnes/notion-sdk-py", + "expected": "custom_message", + "got": "custom_message", + "got_raw": "custom_message", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ramnes/notion-sdk-py", + "expected": "HTTPResponseError)", + "got": "HTTPResponseError)", + "got_raw": "HTTPResponseError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rcmdnk/homebrew-file", + "expected": "b", + "got": "b", + "got_raw": "b", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rcmdnk/homebrew-file", + "expected": "'", + "got": "''", + "got_raw": "''", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "rcmdnk/homebrew-file", + "expected": "file", + "got": "file", + "got_raw": "file", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rcmdnk/homebrew-file", + "expected": "lines", + "got": "lines", + "got_raw": "lines", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rcmdnk/homebrew-file", + "expected": "''", + "got": "'git file://test_repo'", + "got_raw": "'git file://test_repo'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "readthedocs/sphinx-autoapi", + "expected": "name", + "got": "name", + "got_raw": "name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "readthedocs/sphinx-autoapi", + "expected": "\"Example module\"", + "got": "\"Submodule documentation\"", + "got_raw": "\"Submodule documentation\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3902439024390244 + }, + { + "repo": "readthedocs/sphinx-autoapi", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "readthedocs/sphinx-autoapi", + "expected": "text", + "got": "text", + "got_raw": "text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"tab1\"})", + "got": "\"SELECT * FROM tab1\")", + "got_raw": "\"SELECT * FROM tab1\")", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.4666666666666667 + }, + { + "repo": "reata/sqllineage", + "expected": "\"DESC FUNCTION abs\")", + "got": "\"describe function spark.sql('select * from table')\")", + "got_raw": "\n \"describe function spark.sql('select * from table')\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.1917808219178082 + }, + { + "repo": "reata/sqllineage", + "expected": "SqlFluffColumn._extract_source_columns(segment_mock)", + "got": "find_from_expression_element(segment_mock)", + "got_raw": "find_from_expression_element(segment_mock)", + "exact_match": false, + "code_bleu": 0.44582686998428134, + "edit_similarity": 0.46808510638297873 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"host\"})", + "got": "\"\"\"", + "got_raw": "\"\"\"\n WITH cte AS (SELECT 1)\n SELECT * FROM cte\n \"\"\"", + "exact_match": false, + "code_bleu": 0.2259005009024612, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"tab1\"})", + "got": "\"\"\"", + "got_raw": "\"\"\"\n SELECT\n a\n FROM\n t1\n \"\"\"", + "exact_match": false, + "code_bleu": 0.2259005009024612, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"dataset.tab1\"), ColumnQualifierTuple(\"col1\", \"dataset.target\"), ), ( ColumnQualifierTuple(\"col1\", \"dataset.tab2\"), ColumnQualifierTuple(\"col1\", \"dataset.target\"), ), ])", + "got": "[ColumnQualifierTuple(\"col1\", \"dataset.tab1\"), ColumnQualifierTuple(\"col1\", \"dataset.tab2\")])", + "got_raw": "[ColumnQualifierTuple(\"col1\", \"dataset.tab1\"), ColumnQualifierTuple(\"col1\", \"dataset.tab2\")])", + "exact_match": false, + "code_bleu": 0.3723993643084831, + "edit_similarity": 0.6305084745762712 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"Production.ScrapReason\"})", + "got": "\"\"\"", + "got_raw": "\"\"\"\n UPDATE t1\n SET t1.a = 1\n FROM t1\n \"\"\")", + "exact_match": false, + "code_bleu": 0.2259005009024612, + "edit_similarity": 0.13333333333333333 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple( \"CASE WHEN col1 = 1 THEN 'V1' WHEN col1 = 2 THEN 'V2' END\", \"tab1\" ), ), ])", + "got": "ColumnQualifierTuple(\"col1\", \"tab2\"))", + "got_raw": "ColumnQualifierTuple(\"col1\", \"tab2\"))", + "exact_match": false, + "code_bleu": 0.304304610361494, + "edit_similarity": 0.42045454545454547 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ), ( ColumnQualifierTuple(\"col2\", \"tab2\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ), ( ColumnQualifierTuple(\"col3\", \"tab2\"), ColumnQualifierTuple(\"col1\", \"tab1\"), ), ])", + "got": "[ColumnQualifierTuple(\"col1\", \"tab1\")])", + "got_raw": "[ColumnQualifierTuple(\"col1\", \"tab1\")])", + "exact_match": false, + "code_bleu": 0.12099658818591286, + "edit_similarity": 0.02097902097902098 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"*\", \"tab2\"), ColumnQualifierTuple(\"*\", \"tab3\"), ), ])", + "got": "[(ColumnQualifierTuple(\"tab2\", \"tab1\"), ColumnQualifierTuple(\"tab3\", \"tab1\"))])", + "got_raw": "[(ColumnQualifierTuple(\"tab2\", \"tab1\"), ColumnQualifierTuple(\"tab3\", \"tab1\"))])", + "exact_match": false, + "code_bleu": 0.34633892034568714, + "edit_similarity": 0.8734177215189873 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"tgt\"})", + "got": "{\"tgt\"})", + "got_raw": "{\"tgt\"})", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"films_all\"})", + "got": "sql)", + "got_raw": "sql)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "reata/sqllineage", + "expected": "[ ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col2\", \"tab1\"), ), ( ColumnQualifierTuple(\"col1\", \"tab2\"), ColumnQualifierTuple(\"col3\", \"tab1\"), ), ])", + "got": "[ColumnQualifierTuple(\"col2\", \"tab1\"), ColumnQualifierTuple(\"col3\", \"tab1\")])", + "got_raw": "[ColumnQualifierTuple(\"col2\", \"tab1\"), ColumnQualifierTuple(\"col3\", \"tab1\")])", + "exact_match": false, + "code_bleu": 0.3636308822718766, + "edit_similarity": 0.6337448559670782 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"tab1\"})", + "got": "\"update test_table set a = 1 where a = 2\")", + "got_raw": "\n \"update test_table set a = 1 where a = 2\")", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.27450980392156865 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"dbname.dto.tablename\"})", + "got": "sql)", + "got_raw": "sql)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.13793103448275862 + }, + { + "repo": "reata/sqllineage", + "expected": "Schema(\"a\")", + "got": "Schema(\"a\")", + "got_raw": "Schema(\"a\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "reata/sqllineage", + "expected": "{\"tab_1\"})", + "got": "\"\"\"", + "got_raw": "\"\"\"\n insert overwrite table t1\n select * from t1 join t2 on t1.id = t2.id\n \"\"\"", + "exact_match": false, + "code_bleu": 0.2259005009024612, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "reata/sqllineage", + "expected": "\"show create table tab1\")", + "got": "\"show create table test_table\", \"show create table test_table\", dialect=dialect", + "got_raw": "\n \"show create table test_table\", \"show create table test_table\", dialect=dialect", + "exact_match": false, + "code_bleu": 0.1561969968460128, + "edit_similarity": 0.4423076923076923 + }, + { + "repo": "reata/sqllineage", + "expected": "{Path(\"examples/src/main/resources/\")})", + "got": "\"\"\"", + "got_raw": "\"\"\"\n SELECT * FROM parquet.`/path/to/file.parquet`\n \"\"\")", + "exact_match": false, + "code_bleu": 0.1290860005156921, + "edit_similarity": 0.09523809523809523 + }, + { + "repo": "reata/sqllineage", + "expected": "NetworkXGraphOperator", + "got": "NetworkXGraphOperator", + "got_raw": "NetworkXGraphOperator", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "rv.status_code", + "got": "rv.status_code", + "got_raw": "rv.status_code", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "302", + "got": "302", + "got_raw": "302", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"link\"", + "got": "\"link\"", + "got_raw": "\"link\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "diff", + "got": "diff", + "got_raw": "diff", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"/home\"", + "got": "\"home.md\"", + "got_raw": "\"home.md\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "redimp/otterwiki", + "expected": "data['preview_content']", + "got": "data", + "got_raw": "data", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.2962962962962963 + }, + { + "repo": "redimp/otterwiki", + "expected": "ftoc", + "got": "ftoc", + "got_raw": "ftoc", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "response.data.decode().lower()", + "got": "response.data.decode()", + "got_raw": "response.data.decode()", + "exact_match": false, + "code_bleu": 0.6363636363636364, + "edit_similarity": 0.8461538461538461 + }, + { + "repo": "redimp/otterwiki", + "expected": "\"Home\"", + "got": "\"home\"", + "got_raw": "\"home\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "redimp/otterwiki", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redimp/otterwiki", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-om-python", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-om-python", + "expected": "hash1", + "got": "hash1", + "got_raw": "hash1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-om-python", + "expected": "\"test@example.com\"", + "got": "\"test@example.com\"", + "got_raw": "\"test@example.com\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-om-python", + "expected": "[\"FT.SEARCH\", model_name, \"(@first_name:{*drew*})\", \"LIMIT\", 0, 1000]", + "got": "\"first_name:*drew*\"", + "got_raw": "\"first_name:*drew*\"", + "exact_match": false, + "code_bleu": 0.038461538461538464, + "edit_similarity": 0.4318181818181818 + }, + { + "repo": "redis/redis-om-python", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-om-python", + "expected": "[\"FT.SEARCH\", model_name, \"@bio_fts:%newb%\", \"LIMIT\", 0, 1000]", + "got": "\"member.bio LIKE %newb%\"", + "got_raw": "\"member.bio LIKE %newb%\"", + "exact_match": false, + "code_bleu": 0.038461538461538464, + "edit_similarity": 0.3023255813953488 + }, + { + "repo": "redis/redis-om-python", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-om-python", + "expected": "Priority.HIGH", + "got": "Priority.HIGH", + "got_raw": "Priority.HIGH", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-om-python", + "expected": "[\"FT.SEARCH\", model_name, \"(@first_name:{An*})\", \"LIMIT\", 0, 1000]", + "got": "\"member.first_name:An*\"", + "got_raw": "\"member.first_name:An*\"", + "exact_match": false, + "code_bleu": 0.038461538461538464, + "edit_similarity": 0.4044943820224719 + }, + { + "repo": "redis/redis-om-python", + "expected": "r.stdout", + "got": "r.stdout", + "got_raw": "r.stdout", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-om-python", + "expected": "-1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "redis/redis-om-python", + "expected": "\"JJ\"", + "got": "\"JJ\"", + "got_raw": "\"JJ\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-py", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-py", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-py", + "expected": "8", + "got": "8", + "got_raw": "8", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-py", + "expected": "5", + "got": "[3]", + "got_raw": "[3]", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "redis/redis-py", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-py", + "expected": "b\"2\"", + "got": "b\"3\"", + "got_raw": "b\"3\"", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.75 + }, + { + "repo": "redis/redis-py", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "redis/redis-py", + "expected": "str(e.value)", + "got": "str(e.value)", + "got_raw": "str(e.value)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-py", + "expected": "b\"OK\"", + "got": "\"OK\"", + "got_raw": "\"OK\"", + "exact_match": false, + "code_bleu": 0.29730177875068026, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "redis/redis-py", + "expected": "CBState.OPEN", + "got": "CBState.OPEN", + "got_raw": "CBState.OPEN", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-py", + "expected": "6", + "got": "6000", + "got_raw": "6000", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "redis/redis-py", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-py", + "expected": "[\"2\"])", + "got": "2)", + "got_raw": "2)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.5 + }, + { + "repo": "redis/redis-py", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "redis/redis-py", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-py", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-py", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-py", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-py", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "redis/redis-py", + "expected": "body", + "got": "body", + "got_raw": "body", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "reubano/meza", + "expected": "ft.afterish(\"1,001\")", + "got": "ft.afterish(\"1001\", \"1\")", + "got_raw": "ft.afterish(\"1001\", \"1\")", + "exact_match": false, + "code_bleu": 0.46713797772820004, + "edit_similarity": 0.8636363636363636 + }, + { + "repo": "reubano/meza", + "expected": "current", + "got": "current", + "got_raw": "current", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "reubano/meza", + "expected": "ft.byte(list(content))", + "got": "ft.byte(StringIO(content))", + "got_raw": "ft.byte(StringIO(content))", + "exact_match": false, + "code_bleu": 0.6580370064762462, + "edit_similarity": 0.7916666666666666 + }, + { + "repo": "reubano/meza", + "expected": "parsed", + "got": "parsed", + "got_raw": "parsed", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "richardkiss/pycoin", + "expected": "\"OP_0\")", + "got": "\"OP_0\")", + "got_raw": "\"OP_0\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "richardkiss/pycoin", + "expected": "wif)", + "got": "wif)", + "got_raw": "wif)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "richardkiss/pycoin", + "expected": "g2_neg)", + "got": "g2_neg)", + "got_raw": "g2_neg)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "richardkiss/pycoin", + "expected": "None)", + "got": "secret_exponent)", + "got_raw": "secret_exponent)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.38095238095238093 + }, + { + "repo": "richardkiss/pycoin", + "expected": "EXPECTED)", + "got": "EXPECTED)", + "got_raw": "EXPECTED)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "richardkiss/pycoin", + "expected": "1)", + "got": "115792089237316195423570985008687907853269984665640564039457584007908834671663)", + "got_raw": "115792089237316195423570985008687907853269984665640564039457584007908834671663)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.04938271604938271 + }, + { + "repo": "richardkiss/pycoin", + "expected": "'')", + "got": "repr(key))", + "got_raw": " repr(key))", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.12244897959183673 + }, + { + "repo": "richardkiss/pycoin", + "expected": "tx.is_solution_ok(idx))", + "got": "txs_in.script_sig)", + "got_raw": "txs_in.script_sig)", + "exact_match": false, + "code_bleu": 0.2272202082191573, + "edit_similarity": 0.3902439024390244 + }, + { + "repo": "richardkiss/pycoin", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "richardkiss/pycoin", + "expected": "m0p.hwif())", + "got": "m0p.hwif())", + "got_raw": "m0p.hwif())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "richardkiss/pycoin", + "expected": "address)", + "got": "address)", + "got_raw": "address)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "richardkiss/pycoin", + "expected": "\"1MwkRkogzBRMehBntgcq2aJhXCXStJTXHT\")", + "got": "\"19Vqc8uLTfUonmxUEZac7fz1M5c5ZZbAii\")", + "got_raw": "\"19Vqc8uLTfUonmxUEZac7fz1M5c5ZZbAii\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.16216216216216217 + }, + { + "repo": "richardkiss/pycoin", + "expected": "[1, 2, 0])", + "got": "[1, 2, 0, 0])", + "got_raw": "[1, 2, 0, 0])", + "exact_match": false, + "code_bleu": 0.7510499815709779, + "edit_similarity": 0.8695652173913043 + }, + { + "repo": "richardkiss/pycoin", + "expected": "b\"\\x7f\")", + "got": "b\"\\x7f\")", + "got_raw": "b\"\\x7f\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "richardkiss/pycoin", + "expected": "'FnSbX2G6VkrCqCLBFPxwbrC9xT7C3rh5ko')", + "got": "tx.txs_out[1].puzzle_script().to_p2pkh())", + "got_raw": "tx.txs_out[1].puzzle_script().to_p2pkh())", + "exact_match": false, + "code_bleu": 0.08225964699966554, + "edit_similarity": 0.10256410256410256 + }, + { + "repo": "richardkiss/pycoin", + "expected": "script)", + "got": "script)", + "got_raw": "script)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "richardkiss/pycoin", + "expected": "\"2607:f8b0:4006:80a:0:0:0:200e\")", + "got": "b = b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x", + "got_raw": "b = b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x", + "exact_match": false, + "code_bleu": 0.011429504536192995, + "edit_similarity": 0.049019607843137254 + }, + { + "repo": "richardkiss/pycoin", + "expected": "wif)", + "got": "wif)", + "got_raw": "wif)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "richardkiss/pycoin", + "expected": "\"7c4f5385050c18aa8df2ba50da566bbab68635999cc99b75124863da1594195b\")", + "got": "\"0000000000000000000000000000000000000000000000000000000000000000\")", + "got_raw": "\"0000000000000000000000000000000000000000000000000000000000000000\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.08955223880597014 + }, + { + "repo": "richardkiss/pycoin", + "expected": "se)", + "got": "se)", + "got_raw": "se)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ricklupton/floweaver", + "expected": "[(\"c\", \"e\", 1)]", + "got": "[(\"b\", \"e\", 1)]", + "got_raw": "[(\"b\", \"e\", 1)]", + "exact_match": false, + "code_bleu": 0.7071067811865475, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "ricklupton/floweaver", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ricklupton/floweaver", + "expected": "[1]", + "got": "[-1]", + "got_raw": "[-1]", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "ricklupton/floweaver", + "expected": "{(\"a\", \"b\")}", + "got": "{(\"a\", \"b\")}", + "got_raw": "{(\"a\", \"b\")}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ricklupton/floweaver", + "expected": "[(\"a\", \"x\")]", + "got": "[(\"a\", \"x\")]", + "got_raw": "[(\"a\", \"x\")]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ricklupton/floweaver", + "expected": "Rules( [ ({\"x\": Includes({\"a\"})}, (\"AB\",)), ({\"x\": Includes({\"b\"})}, (\"AB\", \"BC\")), ({\"x\": Includes({\"c\"})}, (\"BC\",)), ({\"x\": Excludes({\"a\", \"b\", \"c\"})}, ()), ] )", + "got": "Rules([({\"x\": Includes({\"a\", \"b\", \"c\"})}, \"ABC\")])", + "got_raw": "Rules([({\"x\": Includes({\"a\", \"b\", \"c\"})}, \"ABC\")])", + "exact_match": false, + "code_bleu": 0.24780633250126013, + "edit_similarity": 0.42452830188679247 + }, + { + "repo": "ricklupton/floweaver", + "expected": "[link.to_json() for link in data.links]", + "got": "[link.to_json() for link in data.links]", + "got_raw": "[link.to_json() for link in data.links]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ricklupton/floweaver", + "expected": "{ \"source\": \"a\", \"target\": \"b\", \"type\": \"c\", \"time\": \"d\", \"value\": 3, \"data\": { \"value\": 2, }, \"title\": \"link\", \"opacity\": 0.9, \"color\": \"blue\", }", + "got": "{", + "got_raw": "{\n \"source\": \"a\",\n \"target\": \"b\",\n \"type\": \"c\",\n \"time\": \"d\",\n \"link_width\": 3,\n \"data\": {\n \"value\": 2,\n },\n \"title\": \"link\",\n }", + "exact_match": false, + "code_bleu": 0.013827989709333965, + "edit_similarity": 0.013605442176870748 + }, + { + "repo": "ricklupton/floweaver", + "expected": "\"g2\"", + "got": "\"g2\"", + "got_raw": "\"g2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ricklupton/floweaver", + "expected": "[ ( \"a^*\", \"b^*\", (\"*\", \"*\"), {\"measures\": {\"value\": 11}, \"original_flows\": [0, 1], \"bundles\": [0]}, ), ]", + "got": "[", + "got_raw": "[\n ((0, 1), (0, 1), {\"material\": \"m\", \"value\": 4, \"another_measure\": 2}),\n ((0, 1), (0, 1), {\"material\": \"m\", \"value\": 7, \"another_measure\": 1}),\n ]", + "exact_match": false, + "code_bleu": 0.015246245064137449, + "edit_similarity": 0.018867924528301886 + }, + { + "repo": "ricklupton/floweaver", + "expected": "(0, 0)", + "got": "(0, 0)", + "got_raw": "(0, 0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ricklupton/floweaver", + "expected": "{\"a^a1\", \"a^a2\", \"b^b1\"}", + "got": "{\"a\", \"b\"}", + "got_raw": "{\"a\", \"b\"}", + "exact_match": false, + "code_bleu": 0.23190565391078496, + "edit_similarity": 0.5882352941176471 + }, + { + "repo": "ricklupton/floweaver", + "expected": "Ordering( [", + "got": "view_graph.ordering", + "got_raw": "view_graph.ordering", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.4666666666666667 + }, + { + "repo": "ricklupton/floweaver", + "expected": "[(\"c\", \"b\", 2)]", + "got": "[(\"c\", \"b\", 2)]", + "got_raw": "[(\"c\", \"b\", 2)]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ricklupton/floweaver", + "expected": "[node.to_json() for node in data.nodes]", + "got": "[{\"id\": \"a\"}]", + "got_raw": "[{\"id\": \"a\"}]", + "exact_match": false, + "code_bleu": 0.11068981993475528, + "edit_similarity": 0.19230769230769232 + }, + { + "repo": "ricklupton/floweaver", + "expected": "[0]", + "got": "flows[flows[\"source\"] == \"a1\"]", + "got_raw": "flows[flows[\"source\"] == \"a1\"]", + "exact_match": false, + "code_bleu": 0.13950796967929133, + "edit_similarity": 0.12121212121212122 + }, + { + "repo": "ricklupton/floweaver", + "expected": "[\"a\", \"b\", \"c\", \"d\"]", + "got": "[\"a\", \"b\", \"c\", \"d\"]", + "got_raw": "[\"a\", \"b\", \"c\", \"d\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ricklupton/floweaver", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "ricklupton/floweaver", + "expected": "{ \"a^*\", \"b^*\", \"via^m\", \"via^n\", \"c^c1\", \"c^c2\", }", + "got": "{\"a\", \"b\", \"c\", \"via\"}", + "got_raw": "{\"a\", \"b\", \"c\", \"via\"}", + "exact_match": false, + "code_bleu": 0.1194110871372874, + "edit_similarity": 0.4657534246575342 + }, + { + "repo": "ricklupton/floweaver", + "expected": "old_flows", + "got": "old_flows", + "got_raw": "old_flows", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rmartin16/qbittorrent-api", + "expected": "WebSeedsList([])", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.28284271247461906, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "rmartin16/qbittorrent-api", + "expected": "all_dotted_methods", + "got": "all_dotted_methods", + "got_raw": "all_dotted_methods", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rmartin16/qbittorrent-api", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rmartin16/qbittorrent-api", + "expected": "files[0]", + "got": "files[0]", + "got_raw": "files[0]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rmartin16/qbittorrent-api", + "expected": "415", + "got": "415", + "got_raw": "415", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rmartin16/qbittorrent-api", + "expected": "NotFound404Error)", + "got": "NotFound404Error)", + "got_raw": "NotFound404Error)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rmartin16/qbittorrent-api", + "expected": "[ ListEntry({\"two\": \"2\"}), ListEntry({\"three\": \"3\"}), ]", + "got": "list_two", + "got_raw": "list_two", + "exact_match": false, + "code_bleu": 0.025, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "rmartin16/qbittorrent-api", + "expected": "405", + "got": "405", + "got_raw": "405", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rmartin16/qbittorrent-api", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rmartin16/qbittorrent-api", + "expected": "NotImplementedError)", + "got": "NotImplementedError)", + "got_raw": "NotImplementedError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rmartin16/qbittorrent-api", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rq/django-rq", + "expected": "[connection_params_1])", + "got": "[connection_params_1])", + "got_raw": "[connection_params_1])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rq/django-rq", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rq/django-rq", + "expected": "400)", + "got": "100)", + "got_raw": "100)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.75 + }, + { + "repo": "rq/django-rq", + "expected": "404)", + "got": "404)", + "got_raw": "404)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rq/django-rq", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rq/django-rq", + "expected": "unique_configs)", + "got": "unique_configs)", + "got_raw": "unique_configs)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rq/django-rq", + "expected": "\"test3\")", + "got": "\"test3\")", + "got_raw": "\"test3\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rq/django-rq", + "expected": "url.startswith('/admin'))", + "got": "url)", + "got_raw": "url)", + "exact_match": false, + "code_bleu": 0.16988673071467442, + "edit_similarity": 0.27586206896551724 + }, + { + "repo": "rq/django-rq", + "expected": "list)", + "got": "list)", + "got_raw": "list)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rq/django-rq", + "expected": "config.get('DB', 0))", + "got": "config['DB'])", + "got_raw": "config['DB'])", + "exact_match": false, + "code_bleu": 0.18037106415283274, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "prompt_text", + "got": "prompt_text", + "got_raw": "prompt_text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "volume3", + "got": "volume3", + "got_raw": "volume3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "challenge", + "got": "challenge", + "got_raw": "challenge", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "error_msg", + "got": "error_msg", + "got_raw": "error_msg", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "None", + "got": "False", + "got_raw": "False", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "call_args[1]", + "got": "call_args.kwargs", + "got_raw": "call_args.kwargs", + "exact_match": false, + "code_bleu": 0.3388507513536918, + "edit_similarity": 0.6428571428571429 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "Version(\"0.18.0\")", + "got": "Version(\"0.18.0\")", + "got_raw": "Version(\"0.18.0\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "root_dir", + "got": "root_dir", + "got_raw": "root_dir", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "text_content", + "got": "text_content", + "got_raw": "text_content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "\"test.com\"", + "got": "\"test.com\"", + "got_raw": "\"test.com\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "ValueError)", + "got": "ValidationError)", + "got_raw": "ValidationError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "volume1", + "got": "\"/var/lib/nginx/certs\"", + "got_raw": "\"/var/lib/nginx/certs\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.20689655172413793 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "call_kwargs", + "got": "call_kwargs", + "got_raw": "call_kwargs", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rtCamp/Frappe-Manager", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "['-']", + "got": "['-']", + "got_raw": "['-']", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "[ 'file.py', 'amod.py', 'tests/test_amod.py', 'tests/run.py', 'sub/amod.py', 'sub/bmod.py', ])", + "got": "['tests/test_amod.py', 'tests/run.py'])", + "got_raw": "['tests/test_amod.py', 'tests/run.py'])", + "exact_match": false, + "code_bleu": 0.2032530992618459, + "edit_similarity": 0.5864661654135338 + }, + { + "repo": "rubik/radon", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "( res, 120, 6, )", + "got": "res", + "got_raw": "res", + "exact_match": false, + "code_bleu": 0.07432544468767006, + "edit_similarity": 0.3157894736842105 + }, + { + "repo": "rubik/radon", + "expected": "False", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "rubik/radon", + "expected": "NotImplementedError)", + "got": "NotImplementedError)", + "got_raw": "NotImplementedError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "( res[1:], 120, 6, )", + "got": "res", + "got_raw": "res", + "exact_match": false, + "code_bleu": 0.049550296458446705, + "edit_similarity": 0.2608695652173913 + }, + { + "repo": "rubik/radon", + "expected": "AttributeError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "rubik/radon", + "expected": "expected_rank)", + "got": "expected_rank)", + "got_raw": "expected_rank)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "rubik/radon", + "expected": "'{}'", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "run-llama/llama_deploy", + "expected": "\"http://localhost:4501/deployments/a_deployment/tasks\")", + "got": "httpx.Response)", + "got_raw": "httpx.Response)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.3142857142857143 + }, + { + "repo": "run-llama/llama_deploy", + "expected": "120.0", + "got": "30", + "got_raw": "30", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2857142857142857 + }, + { + "repo": "run-llama/llama_deploy", + "expected": "caplog.text", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/llama_deploy", + "expected": "500", + "got": "500", + "got_raw": "500", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/llama_deploy", + "expected": "\"session1\"", + "got": "\"session1\"", + "got_raw": "\"session1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/llama_deploy", + "expected": "actual.input", + "got": "actual.input", + "got_raw": "actual.input", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/llama_deploy", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/llama_deploy", + "expected": "( \"https://example.com/llama_deploy.git\", \"branch_name\", )", + "got": "\"https://example.com/llama_deploy.git\"", + "got_raw": "\"https://example.com/llama_deploy.git\"", + "exact_match": false, + "code_bleu": 0.09910059291689341, + "edit_similarity": 0.7916666666666666 + }, + { + "repo": "run-llama/llama_deploy", + "expected": "KeyError)", + "got": "KeyError)", + "got_raw": "KeyError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/llama_deploy", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/llama_deploy", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "run-llama/llama_deploy", + "expected": "mock.ANY)", + "got": "tmp_path)", + "got_raw": "tmp_path)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "run-llama/llama_deploy", + "expected": "\"Ho ricevuto:bar\"", + "got": "\"I have received:bar\"", + "got_raw": "\"I have received:bar\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5789473684210527 + }, + { + "repo": "run-llama/notebookllama", + "expected": "\"host\"", + "got": "\"speaker1\"", + "got_raw": "\"speaker1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.375 + }, + { + "repo": "run-llama/notebookllama", + "expected": "\"Hello World\"", + "got": "\"Hello World\"", + "got_raw": "\"Hello World\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/notebookllama", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/notebookllama", + "expected": "500.0", + "got": "500.0", + "got_raw": "500.0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/notebookllama", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/workflows-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/workflows-py", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/workflows-py", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/workflows-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/workflows-py", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/workflows-py", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/workflows-py", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/workflows-py", + "expected": "\"foo\"", + "got": "\"streaming\"", + "got_raw": "\"streaming\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.25 + }, + { + "repo": "run-llama/workflows-py", + "expected": "[\"slow\"]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.39640237166757364, + "edit_similarity": 0.4 + }, + { + "repo": "run-llama/workflows-py", + "expected": "\"42\"", + "got": "42", + "got_raw": "42", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "run-llama/workflows-py", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/workflows-py", + "expected": "Repo", + "got": "Repo", + "got_raw": "Repo", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/workflows-py", + "expected": "\"ok_2\"", + "got": "\"ok_2\"", + "got_raw": "\"ok_2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/workflows-py", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/workflows-py", + "expected": "schema", + "got": "schema", + "got_raw": "schema", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/workflows-py", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/workflows-py", + "expected": "t2", + "got": "t2", + "got_raw": "t2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/workflows-py", + "expected": "4", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "run-llama/workflows-py", + "expected": "semver", + "got": "semver", + "got_raw": "semver", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "run-llama/workflows-py", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "TimeoutError)", + "got": "TimeoutError)", + "got_raw": "TimeoutError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "0", + "got": "50", + "got_raw": "50", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "200 * 1024 * 1024", + "got": "100 * 1024 * 1024", + "got_raw": "100 * 1024 * 1024", + "exact_match": false, + "code_bleu": 0.7598356856515925, + "edit_similarity": 0.9411764705882353 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "100", + "got": "100", + "got_raw": "100", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "8", + "got": "8", + "got_raw": "8", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "50", + "got": "100", + "got_raw": "100", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "\"zone1\"", + "got": "\"zone1\"", + "got_raw": "\"zone1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "0.1", + "got": "0.1", + "got_raw": "0.1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "60", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "404", + "got": "404", + "got_raw": "404", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ruvnet/wifi-densepose", + "expected": "chunk_size", + "got": "chunk_size", + "got_raw": "chunk_size", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "safety-research/bloom", + "expected": "v", + "got": "v", + "got_raw": "v", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "safety-research/bloom", + "expected": "\"Hi\"", + "got": "\"Hi\"", + "got_raw": "\"Hi\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "safety-research/bloom", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "safety-research/bloom", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sammchardy/python-binance", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sammchardy/python-binance", + "expected": "\"/fapi/v1/accountconfig\"", + "got": "\"/fapi/v1/accountConfig\"", + "got_raw": "\"/fapi/v1/accountConfig\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9583333333333334 + }, + { + "repo": "sammchardy/python-binance", + "expected": "300", + "got": "300", + "got_raw": "300", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sammchardy/python-binance", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sammchardy/python-binance", + "expected": "ts1", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sammchardy/python-binance", + "expected": "logging.DEBUG", + "got": "logging.DEBUG", + "got_raw": "logging.DEBUG", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sammchardy/python-binance", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sammchardy/python-binance", + "expected": "headers", + "got": "headers", + "got_raw": "headers", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sammchardy/python-binance", + "expected": "ws.MAX_RECONNECTS", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.0 + }, + { + "repo": "sammchardy/python-binance", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sammchardy/python-binance", + "expected": "13", + "got": "13", + "got_raw": "13", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sammchardy/python-binance", + "expected": "\"/fapi/v3/positionrisk\"", + "got": "\"/fapi/v3/positionRisk\"", + "got_raw": "\"/fapi/v3/positionRisk\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9565217391304348 + }, + { + "repo": "sammchardy/python-binance", + "expected": "\"error\"", + "got": "\"error\"", + "got_raw": "\"error\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sammchardy/python-binance", + "expected": "order)", + "got": "order)", + "got_raw": "order)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sammchardy/python-binance", + "expected": "'trade'", + "got": "'trade'", + "got_raw": "'trade'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sammchardy/python-binance", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "samuelcolvin/watchfiles", + "expected": "{(Change.added, 'foo.py'), (Change.added, 'spam.md')}", + "got": "IsTuple(", + "got_raw": "IsTuple(\n (Change.added, 'spam.txt'), (Change.added, 'spam.md'), (Change.added, 'foo.py')\n )", + "exact_match": false, + "code_bleu": 0.06320999598962011, + "edit_similarity": 0.06557377049180328 + }, + { + "repo": "samuelcolvin/watchfiles", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "samuelcolvin/watchfiles", + "expected": "[Path('/a/b')]", + "got": "[Path('/a/b')]", + "got_raw": "[Path('/a/b')]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "samuelcolvin/watchfiles", + "expected": "False)", + "got": "1000)", + "got_raw": "1000)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "samuelcolvin/watchfiles", + "expected": "RuntimeError, match='custom exit')", + "got": "RuntimeError)", + "got_raw": "RuntimeError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5531914893617021 + }, + { + "repo": "samuelcolvin/watchfiles", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "samuelcolvin/watchfiles", + "expected": "SysError)", + "got": "SystemExit)", + "got_raw": "SystemExit)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "sanic-org/sanic", + "expected": "\"\u2713\"", + "got": "\"\u2713\"", + "got_raw": "\"\u2713\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sanic-org/sanic", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sanic-org/sanic", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sanic-org/sanic", + "expected": "\"\")", + "got": "access)", + "got_raw": "access)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.2 + }, + { + "repo": "sanic-org/sanic", + "expected": "\"1\"", + "got": "\"1\"", + "got_raw": "\"1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sanic-org/sanic", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sanic-org/sanic", + "expected": "\"no\")", + "got": "\"no\")", + "got_raw": "\"no\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sanic-org/sanic", + "expected": "sloads", + "got": "sloads", + "got_raw": "sloads", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sanic-org/sanic", + "expected": "id(bp3)", + "got": "id(bp3)", + "got_raw": "id(bp3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sanic-org/sanic", + "expected": "logo", + "got": "logo", + "got_raw": "logo", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sanic-org/sanic", + "expected": "\"\"", + "got": "\"127.0.1.1\"", + "got_raw": "\"127.0.1.1\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "sanic-org/sanic", + "expected": "\"OK\"", + "got": "\"OK\"", + "got_raw": "\"OK\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sanic-org/sanic", + "expected": "Stage.IDLE", + "got": "Stage.INIT", + "got_raw": "Stage.INIT", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.7 + }, + { + "repo": "sanic-org/sanic", + "expected": "b\"\"", + "got": "b\"\"", + "got_raw": "b\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sanic-org/sanic", + "expected": "\"get\"", + "got": "\"get\"", + "got_raw": "\"get\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sanic-org/sanic", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sanic-org/sanic", + "expected": "\"OK\"", + "got": "\"OK\"", + "got_raw": "\"OK\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sanic-org/sanic", + "expected": "1", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sanic-org/sanic", + "expected": "b\"..\"", + "got": "b\"..\"", + "got_raw": "b\"..\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sanic-org/sanic", + "expected": "5", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "scanapi/scanapi", + "expected": "\"foo.yaml\")", + "got": "preferences)", + "got_raw": "preferences)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.17391304347826086 + }, + { + "repo": "scanapi/scanapi", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scanapi/scanapi", + "expected": "results", + "got": "results", + "got_raw": "results", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scanapi/scanapi", + "expected": "\"PackageLoader\"", + "got": "\"PackageLoader\"", + "got_raw": "\"PackageLoader\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scanapi/scanapi", + "expected": "sequence", + "got": "expected", + "got_raw": "expected", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "scanapi/scanapi", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scanapi/scanapi", + "expected": "{\"parent_foo\": \"parent_bar\", \"child_foo\": \"child_bar\"}", + "got": "{\"parent_foo\": \"parent_bar\", \"child_foo\": \"child_bar\"}", + "got_raw": "{\"parent_foo\": \"parent_bar\", \"child_foo\": \"child_bar\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scanapi/scanapi", + "expected": "ExitCode.REQUEST_ERROR", + "got": "ExitCode.REQUEST_ERROR", + "got_raw": "ExitCode.REQUEST_ERROR", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scanapi/scanapi", + "expected": "\"http://foo.com/api/foo\"", + "got": "base_path + \"/foo\"", + "got_raw": "base_path + \"/foo\"", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "scanapi/scanapi", + "expected": "7", + "got": "7", + "got_raw": "7", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scanapi/scanapi", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scanapi/scanapi", + "expected": "{\"abc\": \"def\", \"xxx\": \"www\"}", + "got": "endpoint_headers | headers", + "got_raw": "endpoint_headers | headers", + "exact_match": false, + "code_bleu": 0.12663928094193208, + "edit_similarity": 0.18518518518518517 + }, + { + "repo": "scanapi/scanapi", + "expected": "True)", + "got": "called_with(string)", + "got_raw": "called_with(string)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "scanapi/scanapi", + "expected": "{\"parent_foo\": \"parent_bar\"}", + "got": "{\"parent_foo\": \"parent_bar\"}", + "got_raw": "{\"parent_foo\": \"parent_bar\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scanapi/scanapi", + "expected": "[\"./\"]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.39640237166757364, + "edit_similarity": 0.5 + }, + { + "repo": "scanapi/scanapi", + "expected": "\"{'name': 'foo'}\"", + "got": "\"{'name': 'foo'}\"", + "got_raw": "\"{'name': 'foo'}\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scanapi/scanapi", + "expected": "True)", + "got": "True)", + "got_raw": "True)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scanapi/scanapi", + "expected": "len(endpoints)", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.0 + }, + { + "repo": "scanapi/scanapi", + "expected": "{\"abc\": \"123\"}", + "got": "{\"abc\": \"123\"}", + "got_raw": "{\"abc\": \"123\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "(6, 4)", + "got": "(6, 4)", + "got_raw": "(6, 4)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "labels)", + "got": "labels)", + "got_raw": "labels)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "(6,))", + "got": "(3,))", + "got_raw": "(3,))", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "(5,))", + "got": "(10,))", + "got_raw": "(10,))", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "(4,))", + "got": "(3,))", + "got_raw": "(3,))", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "0)", + "got": "3)", + "got_raw": "3)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "len(y))", + "got": "len(y))", + "got_raw": "len(y))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "np.uint8", + "got": "np.uint32", + "got_raw": "np.uint32", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.8235294117647058 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "0)", + "got": "2)", + "got_raw": "2)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "len(mols))", + "got": "len(mols))", + "got_raw": "len(mols))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "(1,))", + "got": "(3,))", + "got_raw": "(3,))", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "len(y))", + "got": "len(y))", + "got_raw": "len(y))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "np.uint32", + "got": "np.uint32", + "got_raw": "np.uint32", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "(8, 2)", + "got": "(10, 0)", + "got_raw": "(10, 0)", + "exact_match": false, + "code_bleu": 0.32466791547509893, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "len(y))", + "got": "len(y))", + "got_raw": "len(y))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "X_skfp_2)", + "got": "X_skfp_2)", + "got_raw": "X_skfp_2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "(6, 3, 1)", + "got": "(3, 2, 1)", + "got_raw": "(3, 2, 1)", + "exact_match": false, + "code_bleu": 0.42728700639623407, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "X_rdkit)", + "got": "X_rdkit)", + "got_raw": "X_rdkit)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "len(y))", + "got": "len(y))", + "got_raw": "len(y))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "scikit-fingerprints/scikit-fingerprints", + "expected": "valid", + "got": "valid", + "got_raw": "valid", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "se2p/pynguin", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "se2p/pynguin", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "se2p/pynguin", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "se2p/pynguin", + "expected": "0", + "got": "test_case.get_statement(1)", + "got_raw": "test_case.get_statement(1)", + "exact_match": false, + "code_bleu": 0.18575057999133598, + "edit_similarity": 0.0 + }, + { + "repo": "se2p/pynguin", + "expected": "6", + "got": "5", + "got_raw": "5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "se2p/pynguin", + "expected": "{}", + "got": "{\"foo\": \"bar\"}", + "got_raw": "{\"foo\": \"bar\"}", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.25 + }, + { + "repo": "se2p/pynguin", + "expected": "conf", + "got": "conf", + "got_raw": "conf", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "se2p/pynguin", + "expected": "goals", + "got": "goals", + "got_raw": "goals", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "se2p/pynguin", + "expected": "[]", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.0 + }, + { + "repo": "se2p/pynguin", + "expected": "rand", + "got": "rand < 50", + "got_raw": "rand < 50", + "exact_match": true, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "se2p/pynguin", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "se2p/pynguin", + "expected": "new", + "got": "new", + "got_raw": "new", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "se2p/pynguin", + "expected": "set()", + "got": "{target}", + "got_raw": "{target}", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.3076923076923077 + }, + { + "repo": "se2p/pynguin", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "se2p/pynguin", + "expected": "foo", + "got": "foo", + "got_raw": "foo", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "se2p/pynguin", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "se2p/pynguin", + "expected": "op)", + "got": "0)", + "got_raw": "0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4 + }, + { + "repo": "se2p/pynguin", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "se2p/pynguin", + "expected": "goal", + "got": "goal", + "got_raw": "goal", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "se2p/pynguin", + "expected": "\"int\"", + "got": "\"int\"", + "got_raw": "\"int\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shakedzy/dython", + "expected": "result['metrics']", + "got": "result", + "got_raw": "result", + "exact_match": false, + "code_bleu": 0.14865088937534013, + "edit_similarity": 0.5217391304347826 + }, + { + "repo": "shakedzy/dython", + "expected": "\"Sepal Length\"", + "got": "\"Sepal Length\"", + "got_raw": "\"Sepal Length\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shakedzy/dython", + "expected": "0.55)", + "got": "0.0)", + "got_raw": "0.0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "shakedzy/dython", + "expected": "corr_mat.shape", + "got": "(3, 3)", + "got_raw": "(3, 3)", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.0 + }, + { + "repo": "shakedzy/dython", + "expected": "1.0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "shakedzy/dython", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shakedzy/dython", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shakedzy/dython", + "expected": "len(thresholds)", + "got": "len(pct2)", + "got_raw": "len(pct2)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.5 + }, + { + "repo": "shakedzy/dython", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shakedzy/dython", + "expected": "result['ks_stat']", + "got": "result['ks_stat']", + "got_raw": "result['ks_stat']", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shakedzy/dython", + "expected": "df", + "got": "df", + "got_raw": "df", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "types", + "got": "types", + "got_raw": "types", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "\"go\"", + "got": "\"go\"", + "got_raw": "\"go\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "data", + "got": "data", + "got_raw": "data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "part.content", + "got": "part.content", + "got_raw": "part.content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "types", + "got": "types", + "got_raw": "types", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "md", + "got": "md", + "got_raw": "md", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "output", + "got": "output", + "got_raw": "output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "shotgun-sh/shotgun", + "expected": "4", + "got": "4", + "got_raw": "4", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "facet)", + "got": "facet)", + "got_raw": "facet)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "4.6 * 1024 ** 2)", + "got": "16 * 1024 ** 2)", + "got_raw": "16 * 1024 ** 2)", + "exact_match": false, + "code_bleu": 0.8408964152537145, + "edit_similarity": 0.9032258064516129 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "4)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "'INF')", + "got": "'inf')", + "got_raw": "'inf')", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "2)", + "got": "0)", + "got_raw": "0)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "len(ordered_types)", + "got": "len(ordered_types)", + "got_raw": "len(ordered_types)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "9)", + "got": "9)", + "got_raw": "9)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "'')", + "got": "'')", + "got_raw": "'')", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "0)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "2.6 * 1024 ** 2)", + "got": "4.4 * 1024 ** 2)", + "got_raw": "4.4 * 1024 ** 2)", + "exact_match": false, + "code_bleu": 0.8408964152537145, + "edit_similarity": 0.875 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "7)", + "got": "2)", + "got_raw": "2)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "schema)", + "got": "schema)", + "got_raw": "schema)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissaschool/xmlschema", + "expected": "str)", + "got": "str)", + "got_raw": "str)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissbruecker/linkding", + "expected": "self.user)", + "got": "self.user)", + "got_raw": "self.user)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissbruecker/linkding", + "expected": "form)", + "got": "form)", + "got_raw": "form)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissbruecker/linkding", + "expected": "bookmark2.tags.all())", + "got": "bookmark2.tags.all())", + "got_raw": "bookmark2.tags.all())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissbruecker/linkding", + "expected": "self.get_text(merge_tags_group))", + "got": "self.get_text(merge_tags_group))", + "got_raw": "self.get_text(merge_tags_group))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissbruecker/linkding", + "expected": "user.has_usable_password())", + "got": "user.check_password(\"password123\"))", + "got_raw": "user.check_password(\"password123\"))", + "exact_match": false, + "code_bleu": 0.32172944208038085, + "edit_similarity": 0.6129032258064516 + }, + { + "repo": "sissbruecker/linkding", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissbruecker/linkding", + "expected": "[])", + "got": "[tag1])", + "got_raw": "[tag1])", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6 + }, + { + "repo": "sissbruecker/linkding", + "expected": "\"tag1\")", + "got": "\"tag1\")", + "got_raw": "\"tag1\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissbruecker/linkding", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissbruecker/linkding", + "expected": "[form[\"sort\"]])", + "got": "[])", + "got_raw": "[])", + "exact_match": false, + "code_bleu": 0.27383275896268816, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "sissbruecker/linkding", + "expected": "radio.has_attr(\"checked\"))", + "got": "radio.has_attr(\"checked\"))", + "got_raw": "radio.has_attr(\"checked\"))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissbruecker/linkding", + "expected": "second_tag.id)", + "got": "second_tag.id)", + "got_raw": "second_tag.id)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissbruecker/linkding", + "expected": "\"text/html\")", + "got": "\"text/html\")", + "got_raw": "\"text/html\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissbruecker/linkding", + "expected": "self.user)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.2686424829558855, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "sissbruecker/linkding", + "expected": "data[\"bookmark\"])", + "got": "data[\"bookmark_id\"])", + "got_raw": "data[\"bookmark_id\"])", + "exact_match": false, + "code_bleu": 0.45180100180492244, + "edit_similarity": 0.918918918918919 + }, + { + "repo": "sissbruecker/linkding", + "expected": "len(token_value) > 0)", + "got": "token_value)", + "got_raw": "token_value)", + "exact_match": false, + "code_bleu": 0.20203050891044216, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "sissbruecker/linkding", + "expected": "data[\"id\"])", + "got": "data['id'])", + "got_raw": "data['id'])", + "exact_match": false, + "code_bleu": 0.45180100180492244, + "edit_similarity": 0.8181818181818182 + }, + { + "repo": "sissbruecker/linkding", + "expected": "count)", + "got": "count)", + "got_raw": "count)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissbruecker/linkding", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sissbruecker/linkding", + "expected": "href)", + "got": "href)", + "got_raw": "href)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "True)", + "got": "True)", + "got_raw": "True)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "adam.adam_1.if_expression())", + "got": "adam.adam_1.if_expression(1, 2, 3))", + "got_raw": "adam.adam_1.if_expression(1, 2, 3))", + "exact_match": false, + "code_bleu": 0.4785543921093738, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "0)", + "got": "10)", + "got_raw": "10)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.8 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "adam.adam_1.bool_or())", + "got": "adam.adam_1.bool_or(True, True))", + "got_raw": "adam.adam_1.bool_or(True, True))", + "exact_match": false, + "code_bleu": 0.5706745777055999, + "edit_similarity": 0.8148148148148148 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "initial_num_work_items", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "count", + "got": "count", + "got_raw": "count", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sixty-north/cosmic-ray", + "expected": "100.0", + "got": "0.5", + "got_raw": "0.5", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "slackapi/bolt-python", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "slackapi/bolt-python", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "slackapi/bolt-python", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "slackapi/bolt-python", + "expected": "2)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "slackapi/bolt-python", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "slackapi/bolt-python", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "slackapi/bolt-python", + "expected": "\"\"", + "got": "\"ok\"", + "got_raw": "\"ok\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "slackapi/bolt-python", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "slackapi/bolt-python", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "slackapi/bolt-python", + "expected": "\"a\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "slackapi/bolt-python", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "slackapi/bolt-python", + "expected": "resp.body", + "got": "resp.body", + "got_raw": "resp.body", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "slackapi/bolt-python", + "expected": "self)", + "got": "self)", + "got_raw": "self)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "slackapi/bolt-python", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "slackapi/bolt-python", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "slackapi/bolt-python", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "slackapi/bolt-python", + "expected": "2)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "slackapi/bolt-python", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "slackapi/bolt-python", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "slackapi/bolt-python", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sloria/TextBlob", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sloria/TextBlob", + "expected": "\",\"", + "got": "\",\"", + "got_raw": "\",\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sloria/TextBlob", + "expected": "\"lenses\"", + "got": "\"lenses\"", + "got_raw": "\"lenses\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sloria/TextBlob", + "expected": "float", + "got": "float", + "got_raw": "float", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sloria/TextBlob", + "expected": "noun_phrases", + "got": "noun_phrases", + "got_raw": "noun_phrases", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sloria/TextBlob", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "agents_list", + "got": "agents_list", + "got_raw": "agents_list", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "\"an AI designed to browse bake a cake.\"", + "got": "\"an AI designed to browse bake a cake.\"", + "got_raw": "\"an AI designed to browse bake a cake.\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "\"Goal 4: Wake up\"", + "got": "\"Goal 4: Wake up\"", + "got_raw": "\"Goal 4: Wake up\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "\"gpt-3.5-turbo\"", + "got": "\"gpt-3.5-turbo\"", + "got_raw": "\"gpt-3.5-turbo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "caplog.text", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "agents", + "got": "agents", + "got_raw": "agents", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "file_content", + "got": "file_content()", + "got_raw": "file_content()", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "20", + "got": "20", + "got_raw": "20", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "KeyError)", + "got": "KeyError)", + "got_raw": "KeyError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "self.generator.performance_evaluation)", + "got": "self.generator.performance_evaluation)", + "got_raw": "self.generator.performance_evaluation)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "ValueError)", + "got": "requests.exceptions.MissingSchema)", + "got_raw": "requests.exceptions.MissingSchema)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.17777777777777778 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "\"[]\"", + "got": "\"[]\"", + "got_raw": "\"[]\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "GPT_3_MODEL", + "got": "GPT_3_MODEL", + "got_raw": "GPT_3_MODEL", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "snap-stanford/MLAgentBench", + "expected": "ai_belief", + "got": "location", + "got_raw": "location", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "prop", + "got": "prop", + "got_raw": "prop", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "url", + "got": "url", + "got_raw": "url", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "child.id", + "got": "child.id", + "got_raw": "child.id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "\"\"", + "got": "credentials", + "got_raw": "credentials", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "prefix", + "got": "prefix", + "got_raw": "prefix", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "\"pat\"", + "got": "\"pat\"", + "got_raw": "\"pat\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "tool", + "got": "tool", + "got_raw": "tool", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "schema", + "got": "schema", + "got_raw": "schema", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "value", + "got": "value", + "got_raw": "value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "caplog.text", + "got": "caplog.text", + "got_raw": "caplog.text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sooperset/mcp-atlassian", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "soxoj/maigret", + "expected": "[]", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "soxoj/maigret", + "expected": "'GitHub'", + "got": "'GitHub'", + "got_raw": "'GitHub'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "soxoj/maigret", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "soxoj/maigret", + "expected": "{ 'ida123': 'username' }", + "got": "{'ida123': 'vk_id'}", + "got_raw": "{'ida123': 'vk_id'}", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6511627906976745 + }, + { + "repo": "soxoj/maigret", + "expected": "report_text", + "got": "report_text", + "got_raw": "report_text", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "soxoj/maigret", + "expected": "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]", + "got": "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]", + "got_raw": "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "soxoj/maigret", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "soxoj/maigret", + "expected": "['username1', 'username2']", + "got": "['username1', 'username2']", + "got_raw": "['username1', 'username2']", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "speaches-ai/speaches", + "expected": "\"custom-whisper-model\"", + "got": "\"custom-whisper-model\"", + "got_raw": "\"custom-whisper-model\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "speaches-ai/speaches", + "expected": "sample_rate", + "got": "sample_rate", + "got_raw": "sample_rate", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "speaches-ai/speaches", + "expected": "\"list\"", + "got": "\"list\"", + "got_raw": "\"list\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "speaches-ai/speaches", + "expected": "200", + "got": "200", + "got_raw": "200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "speaches-ai/speaches", + "expected": "\"test-model\"", + "got": "\"test-model\"", + "got_raw": "\"test-model\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "speaches-ai/speaches", + "expected": "16", + "got": "16", + "got_raw": "16", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "speaches-ai/speaches", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "speaches-ai/speaches", + "expected": "\"text/vtt; charset=utf-8\"", + "got": "\"application/x-vtt+xml\"", + "got_raw": "\"application/x-vtt+xml\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2916666666666667 + }, + { + "repo": "speaches-ai/speaches", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "spylang/spy", + "expected": "\"test\")", + "got": "\"\\x1b[38;5;15m\")", + "got_raw": "\"\\x1b[38;5;15m\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.2608695652173913 + }, + { + "repo": "spylang/spy", + "expected": "d2", + "got": "d2", + "got_raw": "d2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "spylang/spy", + "expected": "\"red\"", + "got": "\"red\"", + "got_raw": "\"red\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "spylang/spy", + "expected": "\"test\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "spylang/spy", + "expected": "\"mod1\"", + "got": "\"mod1\"", + "got_raw": "\"mod1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "spylang/spy", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "spylang/spy", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "spylang/spy", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "spylang/spy", + "expected": "hash(b)", + "got": "hash(b)", + "got_raw": "hash(b)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "spylang/spy", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "spylang/spy", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "spylang/spy", + "expected": "[1, 2, 3]", + "got": "[1, 2, 3]", + "got_raw": "[1, 2, 3]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "spylang/spy", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "spylang/spy", + "expected": "\"a.b.c\"", + "got": "\"a.b.c\"", + "got_raw": "\"a.b.c\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "spylang/spy", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "spylang/spy", + "expected": "\"foo\"", + "got": "\"foo\"", + "got_raw": "\"foo\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "spylang/spy", + "expected": "(1, 1)", + "got": "Point(1, 1)", + "got_raw": "Point(1, 1)", + "exact_match": false, + "code_bleu": 0.8091067115702212, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "spylang/spy", + "expected": "w_t", + "got": "w_t", + "got_raw": "w_t", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "spylang/spy", + "expected": "1", + "got": "ANYTHING", + "got_raw": "ANYTHING", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "spylang/spy", + "expected": "B.w_i32", + "got": "i32", + "got_raw": "i32", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.6 + }, + { + "repo": "sql-hkr/tiny8", + "expected": "initial_pc", + "got": "initial_pc", + "got_raw": "initial_pc", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sql-hkr/tiny8", + "expected": "0)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "sql-hkr/tiny8", + "expected": "42", + "got": "42", + "got_raw": "42", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sql-hkr/tiny8", + "expected": "{0: 100, 1: 200})", + "got": "{ 'r0': 999, 'r1': 200 })", + "got_raw": "{ 'r0': 999, 'r1': 200 })", + "exact_match": false, + "code_bleu": 0.39458812555917666, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "sql-hkr/tiny8", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sql-hkr/tiny8", + "expected": "0xFF", + "got": "0x00", + "got_raw": "0x00", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "sql-hkr/tiny8", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sql-hkr/tiny8", + "expected": "1", + "got": "99", + "got_raw": "99", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sql-hkr/tiny8", + "expected": "16", + "got": "16", + "got_raw": "16", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sql-hkr/tiny8", + "expected": "1", + "got": "99", + "got_raw": "99", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "sql-hkr/tiny8", + "expected": "15)", + "got": "15)", + "got_raw": "15)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sqlalchemy/alembic", + "expected": "_down(\"a\", None, True))", + "got": "self.updater.delete_multi_match())", + "got_raw": "self.updater.delete_multi_match())", + "exact_match": false, + "code_bleu": 0.20159257283470225, + "edit_similarity": 0.17543859649122806 + }, + { + "repo": "sqlalchemy/alembic", + "expected": "buf.getvalue()", + "got": "buf.getvalue()", + "got_raw": "buf.getvalue()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sqlalchemy/alembic", + "expected": "\"current\")", + "got": "\"edit\")", + "got_raw": "\"edit\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5882352941176471 + }, + { + "repo": "sqlalchemy/alembic", + "expected": "\"DROP INDEX my_idx ON my_table\")", + "got": "text(\"DROP INDEX my_idx ON my_table\"))", + "got_raw": "text(\"DROP INDEX my_idx ON my_table\"))", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.9142857142857143 + }, + { + "repo": "sqlalchemy/alembic", + "expected": "\"upgrade_ops\")", + "got": "u2)", + "got_raw": "u2)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "sqlalchemy/alembic", + "expected": "buf.getvalue()", + "got": "\"\"\"CREATE TABLE sometable (", + "got_raw": "\"\"\"CREATE TABLE sometable (\n data pgenum\n);\"\"\"", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.2926829268292683 + }, + { + "repo": "sqlalchemy/alembic", + "expected": "buf.getvalue()", + "got": "buf.getvalue()", + "got_raw": "buf.getvalue()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sqlalchemy/alembic", + "expected": "\"t1\")", + "got": "ops.drop_table(\"foo\")", + "got_raw": "ops.drop_table(\"foo\")", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.23076923076923078 + }, + { + "repo": "sqlalchemy/alembic", + "expected": "\"base\")", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "sqlalchemy/alembic", + "expected": "buf.getvalue()", + "got": "buf.getvalue()", + "got_raw": "buf.getvalue()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sqlalchemy/alembic", + "expected": "String", + "got": "String", + "got_raw": "String", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sqlalchemy/alembic", + "expected": "\"COMMENT='This is a table comment'\")", + "got": "\"\"\"CREATE TABLE t2 (c1 INT NOT NULL PRIMARY KEY COMMENT 'This is a table comment')\"\"\")", + "got_raw": "\"\"\"CREATE TABLE t2 (c1 INT NOT NULL PRIMARY KEY COMMENT 'This is a table comment')\"\"\")", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.5737704918032787 + }, + { + "repo": "sqlalchemy/alembic", + "expected": "self.m2", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.18181818181818182 + }, + { + "repo": "stabldev/torrra", + "expected": "\"textual-dark\"", + "got": "\"dark\"", + "got_raw": "\"dark\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6 + }, + { + "repo": "stabldev/torrra", + "expected": "840499200", + "got": "840499200", + "got_raw": "840499200", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "45.6", + "got": "45.6", + "got_raw": "45.6", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "stabldev/torrra", + "expected": "\"1.00 TB\"", + "got": "\"5.00 TB\"", + "got_raw": "\"5.00 TB\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "stabldev/torrra", + "expected": "list_result.output", + "got": "list_result.output", + "got_raw": "list_result.output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "stac-utils/stac-fastapi", + "expected": "\"{'op': '=', 'args': [{'property': 'id'}, 'test-item']}\"", + "got": "\"test-item\"", + "got_raw": "\"test-item\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3283582089552239 + }, + { + "repo": "stac-utils/stac-fastapi", + "expected": "conf", + "got": "conf", + "got_raw": "conf", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "stac-utils/stac-fastapi", + "expected": "\"1\"", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "stac-utils/stac-fastapi", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "stac-utils/stac-fastapi", + "expected": "\"application/schema+json\"", + "got": "\"application/json\"", + "got_raw": "\"application/json\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8372093023255814 + }, + { + "repo": "stac-utils/stac-fastapi", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "stac-utils/stac-fastapi", + "expected": "{\"gte\": 95}", + "got": "{\"gte\": 95}", + "got_raw": "{\"gte\": 95}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "stac-utils/stac-fastapi", + "expected": "500", + "got": "422", + "got_raw": "422", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "truncated_actual_output)", + "got": "truncated_actual_output)", + "got_raw": "truncated_actual_output)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "output)", + "got": "output)", + "got_raw": "output)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "test_string", + "got": "test_string", + "got_raw": "test_string", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "stanford-cs336/assignment1-basics", + "expected": "expected_arrays[key])", + "got": "olerance(rtol, atol)", + "got_raw": "olerance(rtol, atol)", + "exact_match": false, + "code_bleu": 0.22089591134157885, + "edit_similarity": 0.24390243902439024 + }, + { + "repo": "steamship-core/python-client", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "steamship-core/python-client", + "expected": "str(e)", + "got": "str(e)", + "got_raw": "str(e)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "steamship-core/python-client", + "expected": "test_doc", + "got": "test_doc", + "got_raw": "test_doc", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "steamship-core/python-client", + "expected": "TEST_STRING", + "got": "TEST_STRING", + "got_raw": "TEST_STRING", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "steamship-core/python-client", + "expected": "new_output", + "got": "new_output", + "got_raw": "new_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "steamship-core/python-client", + "expected": "default.id", + "got": "default.id", + "got_raw": "default.id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "steamship-core/python-client", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "steamship-core/python-client", + "expected": "\"\"", + "got": "instance4.id", + "got_raw": "instance4.id", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.0 + }, + { + "repo": "steamship-core/python-client", + "expected": "workspace.id", + "got": "workspace.id", + "got_raw": "workspace.id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "steamship-core/python-client", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "steamship-core/python-client", + "expected": "RES_EMPTY", + "got": "RES_EMPTY", + "got_raw": "RES_EMPTY", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "steamship-core/python-client", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "steamship-core/python-client", + "expected": "\"apples\"", + "got": "\"Try some apples!\"", + "got_raw": "\"Try some apples!\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "steamship-core/python-client", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "steamship-core/python-client", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "steamship-core/python-client", + "expected": "TEST_DOC", + "got": "TEST_DOC", + "got_raw": "TEST_DOC", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "steamship-core/python-client", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "steamship-core/python-client", + "expected": "t", + "got": "t", + "got_raw": "t", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "steamship-core/python-client", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "steamship-core/python-client", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimehq/package_control", + "expected": "v.check())", + "got": "v)", + "got_raw": "v)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "sublimehq/package_control", + "expected": "check_version(spec, version, True))", + "got": "check_version(version, spec))", + "got_raw": "check_version(version, spec))", + "exact_match": false, + "code_bleu": 0.2889453466531894, + "edit_similarity": 0.8125 + }, + { + "repo": "sublimehq/package_control", + "expected": "is_compatible_version(\"<3176\", 3175))", + "got": "is_compatible_version('1.0', '1.1'))", + "got_raw": "is_compatible_version('1.0', '1.1'))", + "exact_match": false, + "code_bleu": 0.32172944208038085, + "edit_similarity": 0.7671232876712328 + }, + { + "repo": "sublimehq/package_control", + "expected": "resolve_url( \"https://github.com/packagecontrol-test/package_control-tester/repository.json\", \"https://github.com/packagecontrol-test-2/package_control-tester-2\" ))", + "got": "\"https://github.com/ghuser/repo/blob/master/file.txt\")", + "got_raw": "\"https://github.com/ghuser/repo/blob/master/file.txt\")", + "exact_match": false, + "code_bleu": 0.153509990260506, + "edit_similarity": 0.3577981651376147 + }, + { + "repo": "sublimehq/package_control", + "expected": "normalized", + "got": "normalized", + "got_raw": "normalized", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimehq/package_control", + "expected": "did.exists())", + "got": "did.exists())", + "got_raw": "did.exists())", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimehq/package_control", + "expected": "is_compatible_version(\">3176\", 3177))", + "got": "is_compatible_version('1.0.0', '1.0.0'))", + "got_raw": "is_compatible_version('1.0.0', '1.0.0'))", + "exact_match": false, + "code_bleu": 0.32172944208038085, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "sublimehq/package_control", + "expected": "client.user_info(\"https://bitbucket.org/wbond\"))", + "got": "client.user_info())", + "got_raw": "client.user_info())", + "exact_match": false, + "code_bleu": 0.6060915267313265, + "edit_similarity": 0.5671641791044776 + }, + { + "repo": "sublimelsp/LSP", + "expected": "d.get(\"some.nonexistant.key\"))", + "got": "d.get(\"foo.baz\"))", + "got_raw": "d.get(\"foo.baz\"))", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.5531914893617021 + }, + { + "repo": "sublimelsp/LSP", + "expected": "2)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "sublimelsp/LSP", + "expected": "self.session.capabilities)", + "got": "self.sb.capabilities)", + "got_raw": "self.sb.capabilities)", + "exact_match": false, + "code_bleu": 0.488923022434901, + "edit_similarity": 0.851063829787234 + }, + { + "repo": "sublimelsp/LSP", + "expected": "view.is_loading())", + "got": "view.is_read_only())", + "got_raw": "view.is_read_only())", + "exact_match": false, + "code_bleu": 0.488923022434901, + "edit_similarity": 0.7368421052631579 + }, + { + "repo": "sublimelsp/LSP", + "expected": "d.get(\"foo\"))", + "got": "d.get(\"foo\"))", + "got_raw": "d.get(\"foo\"))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "expected_regex)", + "got": "expected_regex)", + "got_raw": "expected_regex)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"F\")", + "got": "'F')", + "got_raw": "'F')", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"bar\\nbar\\nbar\\n\")", + "got": "\"bar\\nbar\\nbar\\n\")", + "got_raw": "\"bar\\nbar\\nbar\\n\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "{})", + "got": "{})", + "got_raw": "{})", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"/short-path\")", + "got": "\"/short-path\")", + "got_raw": "\"/short-path\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "dict())", + "got": "{})", + "got_raw": "{})", + "exact_match": false, + "code_bleu": 0.3388507513536918, + "edit_similarity": 0.2 + }, + { + "repo": "sublimelsp/LSP", + "expected": "self.view.settings().get(\"translate_tabs_to_spaces\"))", + "got": "self.view.settings().get(\"translate_tabs_to_spaces\"))", + "got_raw": "self.view.settings().get(\"translate_tabs_to_spaces\"))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "text)", + "got": "text)", + "got_raw": "text)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "any(manager.match_view(self.view)))", + "got": "DISABLED_CONFIG.enabled)", + "got_raw": "DISABLED_CONFIG.enabled)", + "exact_match": false, + "code_bleu": 0.13254512146117509, + "edit_similarity": 0.1694915254237288 + }, + { + "repo": "sublimelsp/LSP", + "expected": "'c')", + "got": "'c')", + "got_raw": "'c')", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sublimelsp/LSP", + "expected": "overriddes['unknown'])", + "got": "overriddes[\"unknown\"])", + "got_raw": "overriddes[\"unknown\"])", + "exact_match": false, + "code_bleu": 0.45180100180492244, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "sublimelsp/LSP", + "expected": "parse_uri(\"file:///c:/dir ectory/file.txt\")[1])", + "got": "filename_to_uri(\"file:///C:/dir%20ectory/file.txt\"))", + "got_raw": "filename_to_uri(\"file:///C:/dir%20ectory/file.txt\"))", + "exact_match": false, + "code_bleu": 0.20291744717193683, + "edit_similarity": 0.7676767676767676 + }, + { + "repo": "sublimelsp/LSP", + "expected": "10)", + "got": "14)", + "got_raw": "14)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "sublimelsp/LSP", + "expected": "\"fd\")", + "got": "'fd')", + "got_raw": "'fd')", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6 + }, + { + "repo": "sublimelsp/LSP", + "expected": "6)", + "got": "6)", + "got_raw": "6)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sunscrapers/djoser", + "expected": "user.get_username())", + "got": "user.custom_username)", + "got_raw": "user.custom_username)", + "exact_match": false, + "code_bleu": 0.3387551654364098, + "edit_similarity": 0.7804878048780488 + }, + { + "repo": "sunscrapers/djoser", + "expected": "settings.CONSTANTS.messages.INVALID_CREDENTIALS_ERROR)", + "got": "self.signal_sent)", + "got_raw": "self.signal_sent)", + "exact_match": false, + "code_bleu": 0.19881768219176266, + "edit_similarity": 0.19718309859154928 + }, + { + "repo": "sunscrapers/djoser", + "expected": "user.auth_token.key)", + "got": "user.auth_token.key)", + "got_raw": "user.auth_token.key)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sunscrapers/djoser", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sunscrapers/djoser", + "expected": "status.HTTP_400_BAD_REQUEST)", + "got": "status.HTTP_400_BAD_REQUEST)", + "got_raw": "status.HTTP_400_BAD_REQUEST)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sunscrapers/djoser", + "expected": "response.data)", + "got": "response.data)", + "got_raw": "response.data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sunscrapers/djoser", + "expected": "\"TokenSerializer\")", + "got": "\"TokenSerializer\")", + "got_raw": "\"TokenSerializer\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sunscrapers/djoser", + "expected": "status.HTTP_204_NO_CONTENT)", + "got": "status.HTTP_204_NO_CONTENT)", + "got_raw": "status.HTTP_204_NO_CONTENT)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sunscrapers/djoser", + "expected": "status.HTTP_204_NO_CONTENT)", + "got": "status.HTTP_204_NO_CONTENT)", + "got_raw": "status.HTTP_204_NO_CONTENT)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sunscrapers/djoser", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sunscrapers/djoser", + "expected": "user.username)", + "got": "user.username)", + "got_raw": "user.username)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sunscrapers/djoser", + "expected": "ExampleUser)", + "got": "ExampleUser, data[\"email\"])", + "got_raw": "ExampleUser, data[\"email\"])", + "exact_match": false, + "code_bleu": 0.20556680845025982, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "sunscrapers/djoser", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sunscrapers/djoser", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "sunscrapers/djoser", + "expected": "\"my_domain\"", + "got": "\"my_domain\"", + "got_raw": "\"my_domain\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "swar/nba_api", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "swar/nba_api", + "expected": "GAME_ID", + "got": "GAME_ID", + "got_raw": "GAME_ID", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "swar/nba_api", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "swar/nba_api", + "expected": "4", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "swar/nba_api", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "swar/nba_api", + "expected": "\"2024-25\"", + "got": "\"2024-25\"", + "got_raw": "\"2024-25\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "swar/nba_api", + "expected": "2", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "swar/nba_api", + "expected": "\"00\"", + "got": "\"00\"", + "got_raw": "\"00\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "swar/nba_api", + "expected": "play[\"player\"]", + "got": "play[\"player\"]", + "got_raw": "play[\"player\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "swar/nba_api", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "swar/nba_api", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "swar/nba_api", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "swar/nba_api", + "expected": "\"123\"", + "got": "None", + "got_raw": "None", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "swar/nba_api", + "expected": "3", + "got": "\"2025-11-01\"", + "got_raw": "\"2025-11-01\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "swar/nba_api", + "expected": "3", + "got": "\"20251105\"", + "got_raw": "\"20251105\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "swar/nba_api", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "swar/nba_api", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "swar/nba_api", + "expected": "content", + "got": "content", + "got_raw": "content", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "swar/nba_api", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "swar/nba_api", + "expected": "games", + "got": "games", + "got_raw": "games", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "syrupy-project/syrupy", + "expected": "FailedToLoadModuleMember)", + "got": "FailedToLoadModuleMember)", + "got_raw": "FailedToLoadModuleMember)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "syrupy-project/syrupy", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "syrupy-project/syrupy", + "expected": "snapshot_attrs", + "got": "snapshot_attrs(point)", + "got_raw": "snapshot_attrs(point)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.8 + }, + { + "repo": "syrupy-project/syrupy", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "syrupy-project/syrupy", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "syrupy-project/syrupy", + "expected": "snapshot(name=\"case3\", diff=\"large snapshot\")", + "got": "snapshot(name=\"case 3\")", + "got_raw": "snapshot(name=\"case 3\")", + "exact_match": false, + "code_bleu": 0.386075330496457, + "edit_similarity": 0.6470588235294118 + }, + { + "repo": "syrupy-project/syrupy", + "expected": "TypeError, match=\"At least 1 prop name is required.\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.17909498863725698, + "edit_similarity": 0.21875 + }, + { + "repo": "syrupy-project/syrupy", + "expected": "result.stdout.str()", + "got": "result.stdout.str()", + "got_raw": "result.stdout.str()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "syrupy-project/syrupy", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "syrupy-project/syrupy", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "syrupy-project/syrupy", + "expected": "PathTypeError, match=\"argument cannot be empty\")", + "got": "PathTypeError)", + "got_raw": "PathTypeError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.45161290322580644 + }, + { + "repo": "syrupy-project/syrupy", + "expected": "expected_status_code", + "got": "expected_status_code", + "got_raw": "expected_status_code", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "syrupy-project/syrupy", + "expected": "snapshot(diff=\"does not exist index\")", + "got": "snapshot", + "got_raw": "snapshot", + "exact_match": false, + "code_bleu": 0.09910059291689341, + "edit_similarity": 0.35555555555555557 + }, + { + "repo": "syrupy-project/syrupy", + "expected": "snapshot(name=\"test_is_awesome\")", + "got": "snapshot(name=\"test_is_amazing\", suffix=\"awesome\")", + "got_raw": "snapshot(name=\"test_is_amazing\", suffix=\"awesome\")", + "exact_match": false, + "code_bleu": 0.3672056269893592, + "edit_similarity": 0.7804878048780488 + }, + { + "repo": "tableau/server-client-python", + "expected": "all_tables[0].name", + "got": "all_tables[0].name", + "got_raw": "all_tables[0].name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tableau/server-client-python", + "expected": "in_user.id", + "got": "in_user.id", + "got_raw": "in_user.id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tableau/server-client-python", + "expected": "all_alerts[0].id", + "got": "all_alerts[0].id", + "got_raw": "all_alerts[0].id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tableau/server-client-python", + "expected": "views_list[2].id", + "got": "views_list[2].id", + "got_raw": "views_list[2].id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tableau/server-client-python", + "expected": "task.last_run_at", + "got": "task.schedule_item.last_run_at", + "got_raw": "task.schedule_item.last_run_at", + "exact_match": false, + "code_bleu": 0.427287006396234, + "edit_similarity": 0.6956521739130435 + }, + { + "repo": "tableau/server-client-python", + "expected": "\"gs-3\"", + "got": "\"gs31\"", + "got_raw": "\"gs31\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "tableau/server-client-python", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tableau/server-client-python", + "expected": "3", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "tableau/server-client-python", + "expected": "\"hyper\"", + "got": "\"hyper\"", + "got_raw": "\"hyper\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tableau/server-client-python", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tableau/server-client-python", + "expected": "all_tables[0].id", + "got": "all_tables[0].id", + "got_raw": "all_tables[0].id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tableau/server-client-python", + "expected": "slept)", + "got": "30)", + "got_raw": "30)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "tableau/server-client-python", + "expected": "\"2\"", + "got": "\"2\"", + "got_raw": "\"2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tableau/server-client-python", + "expected": "all_tables[1].name", + "got": "all_tables[1].name", + "got_raw": "all_tables[1].name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tableau/server-client-python", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tableau/server-client-python", + "expected": "subscription.id", + "got": "subscription.id", + "got_raw": "subscription.id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tableau/server-client-python", + "expected": "id_", + "got": "id_", + "got_raw": "id_", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tableau/server-client-python", + "expected": "flow.state", + "got": "flow.state", + "got_raw": "flow.state", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tableau/server-client-python", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tableau/server-client-python", + "expected": "ds1.id", + "got": "ds1.id", + "got_raw": "ds1.id", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taskiq-python/taskiq", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taskiq-python/taskiq", + "expected": "ValueError)", + "got": "NoResultError)", + "got_raw": "NoResultError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.56 + }, + { + "repo": "taskiq-python/taskiq", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taskiq-python/taskiq", + "expected": "{\"b\": 2}", + "got": "{\"b\": 2}", + "got_raw": "{\"b\": 2}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taskiq-python/taskiq", + "expected": "\"a\"", + "got": "\"a\"", + "got_raw": "\"a\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taskiq-python/taskiq", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taskiq-python/taskiq", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taskiq-python/taskiq", + "expected": "0)", + "got": "0)", + "got_raw": "0)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taskiq-python/taskiq", + "expected": "\"4\")", + "got": "4)", + "got_raw": "4)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "taskiq-python/taskiq", + "expected": "[1, 2]", + "got": "[1, 2]", + "got_raw": "[1, 2]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taskiq-python/taskiq", + "expected": "state", + "got": "state", + "got_raw": "state", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taskiq-python/taskiq", + "expected": "span_after)", + "got": "span_after)", + "got_raw": "span_after)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taskiq-python/taskiq", + "expected": "asyncio.TimeoutError)", + "got": "asyncio.CancelledError)", + "got_raw": "asyncio.CancelledError)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6818181818181818 + }, + { + "repo": "taskiq-python/taskiq", + "expected": "{\"a\": 1, \"b\": \"f3\"}", + "got": "{\"a\": 1}", + "got_raw": "{\"a\": 1}", + "exact_match": false, + "code_bleu": 0.44181707153725036, + "edit_similarity": 0.5925925925925926 + }, + { + "repo": "taskiq-python/taskiq", + "expected": "retry_count", + "got": "retry_count", + "got_raw": "retry_count", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taverntesting/tavern", + "expected": "TypeError)", + "got": "SystemExit)", + "got_raw": "SystemExit)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.38095238095238093 + }, + { + "repo": "taverntesting/tavern", + "expected": "\"3\"", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "taverntesting/tavern", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taverntesting/tavern", + "expected": "exceptions.MQTTTopicException)", + "got": "exceptions.MissingQueueError)", + "got_raw": "exceptions.MissingQueueError)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.5423728813559322 + }, + { + "repo": "taverntesting/tavern", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taverntesting/tavern", + "expected": "\"\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.47619047619047616 + }, + { + "repo": "taverntesting/tavern", + "expected": "\"My Upload\"", + "got": "\"My Upload\"", + "got_raw": "\"My Upload\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taverntesting/tavern", + "expected": "m._publish_args", + "got": "m", + "got_raw": "m", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.125 + }, + { + "repo": "taverntesting/tavern", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taverntesting/tavern", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taverntesting/tavern", + "expected": "5)", + "got": "5)", + "got_raw": "5)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taverntesting/tavern", + "expected": "x", + "got": "x", + "got_raw": "x", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taverntesting/tavern", + "expected": "parsed", + "got": "parsed", + "got_raw": "parsed", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taverntesting/tavern", + "expected": "StrictSetting.OFF", + "got": "StrictSetting.OFF", + "got_raw": "StrictSetting.OFF", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taverntesting/tavern", + "expected": "False)", + "got": "False)", + "got_raw": "False)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taverntesting/tavern", + "expected": "{\"foo\": \"myzclqkptpk\"}", + "got": "{\"foo\": \"myzclqkptpk\"}", + "got_raw": "{\"foo\": \"myzclqkptpk\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taylorwilsdon/google_workspace_mcp", + "expected": "url_used", + "got": "url_used", + "got_raw": "url_used", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taylorwilsdon/google_workspace_mcp", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taylorwilsdon/google_workspace_mcp", + "expected": "len(payload)", + "got": "50", + "got_raw": "50", + "exact_match": false, + "code_bleu": 0.125, + "edit_similarity": 0.0 + }, + { + "repo": "taylorwilsdon/google_workspace_mcp", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taylorwilsdon/google_workspace_mcp", + "expected": "\"CENTER\"", + "got": "\"CENTER\"", + "got_raw": "\"CENTER\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taylorwilsdon/google_workspace_mcp", + "expected": "response.body", + "got": "response.body", + "got_raw": "response.body", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taylorwilsdon/google_workspace_mcp", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taylorwilsdon/google_workspace_mcp", + "expected": "ValueError, match=\"Unknown service\")", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.46808510638297873 + }, + { + "repo": "taylorwilsdon/google_workspace_mcp", + "expected": "\"HEADING_1\"", + "got": "\"HEADING_1\"", + "got_raw": "\"HEADING_1\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "taylorwilsdon/google_workspace_mcp", + "expected": "{\"gmail\": \"send\"}", + "got": "{\"gmail\": \"send\"}", + "got_raw": "{\"gmail\": \"send\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tefra/xsdata", + "expected": "list(container))", + "got": "list(container))", + "got_raw": "list(container))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tefra/xsdata", + "expected": "Books)", + "got": "Books)", + "got_raw": "Books)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tefra/xsdata", + "expected": "len(actual))", + "got": "len(actual))", + "got_raw": "len(actual))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tefra/xsdata", + "expected": "list(actual))", + "got": "list(actual))", + "got_raw": "list(actual))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tefra/xsdata", + "expected": "len(headers))", + "got": "len(result))", + "got_raw": "len(result))", + "exact_match": false, + "code_bleu": 0.45180100180492244, + "edit_similarity": 0.64 + }, + { + "repo": "tefra/xsdata", + "expected": "actual.bar)", + "got": "actual.bar)", + "got_raw": "actual.bar)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tefra/xsdata", + "expected": "len(target.attrs))", + "got": "len(target))", + "got_raw": "len(target))", + "exact_match": false, + "code_bleu": 0.4316250567652526, + "edit_similarity": 0.8 + }, + { + "repo": "tefra/xsdata", + "expected": "attr.tag)", + "got": "attr.tag)", + "got_raw": "attr.tag)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tefra/xsdata", + "expected": "objects[-1])", + "got": "objects[0])", + "got_raw": "objects[0])", + "exact_match": false, + "code_bleu": 0.3765008348374354, + "edit_similarity": 0.8695652173913043 + }, + { + "repo": "tefra/xsdata", + "expected": "obj.bar)", + "got": "obj.bar)", + "got_raw": "obj.bar)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tefra/xsdata", + "expected": "len(a.inner))", + "got": "len(b.inner))", + "got_raw": "len(b.inner))", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "tefra/xsdata", + "expected": "len(inner.inner))", + "got": "len(inner.inner))", + "got_raw": "len(inner.inner))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tefra/xsdata", + "expected": "actual[0])", + "got": "actual[0])", + "got_raw": "actual[0])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tefra/xsdata", + "expected": "len(item.attrs))", + "got": "len(item.attributes))", + "got_raw": "len(item.attributes))", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.8648648648648649 + }, + { + "repo": "tefra/xsdata", + "expected": "len(target.attrs))", + "got": "target.attrs[0].restrictions.max_occurs)", + "got_raw": "target.attrs[0].restrictions.max_occurs)", + "exact_match": false, + "code_bleu": 0.22416933501922287, + "edit_similarity": 0.4482758620689655 + }, + { + "repo": "tefra/xsdata", + "expected": "len(actual))", + "got": "len(actual))", + "got_raw": "len(actual))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tefra/xsdata", + "expected": "list(obj.bases))", + "got": "obj.bases)", + "got_raw": "obj.bases)", + "exact_match": false, + "code_bleu": 0.5714285714285714, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "tefra/xsdata", + "expected": "value)", + "got": "value)", + "got_raw": "value)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tefra/xsdata", + "expected": "len(target.attrs))", + "got": "len(target.attrs))", + "got_raw": "len(target.attrs))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tefra/xsdata", + "expected": "attr)", + "got": "attr)", + "got_raw": "attr)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "temporal-community/temporal-ai-agent", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "temporal-community/temporal-ai-agent", + "expected": "\"TestTool\"", + "got": "\"TestTool\"", + "got_raw": "\"TestTool\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "temporal-community/temporal-ai-agent", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "temporal-community/temporal-ai-agent", + "expected": "\"confirm\"", + "got": "\"confirm\"", + "got_raw": "\"confirm\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "temporalio/samples-python", + "expected": "(await handle.describe()).status", + "got": "handle.status", + "got_raw": "handle.status", + "exact_match": false, + "code_bleu": 0.21213203435596426, + "edit_similarity": 0.5777777777777777 + }, + { + "repo": "temporalio/samples-python", + "expected": "Language.ENGLISH", + "got": "Language.EN", + "got_raw": "Language.EN", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.8148148148148148 + }, + { + "repo": "temporalio/samples-python", + "expected": "WorkflowFailureError)", + "got": "CancelledError)", + "got_raw": "CancelledError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4444444444444444 + }, + { + "repo": "temporalio/samples-python", + "expected": "await handle.result()", + "got": "await handle.result()).greeting", + "got_raw": "await handle.result()).greeting", + "exact_match": false, + "code_bleu": 0.6389431042462724, + "edit_similarity": 0.8076923076923077 + }, + { + "repo": "temporalio/samples-python", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "temporalio/samples-python", + "expected": "Language.CHINESE", + "got": "Language.CHINESE", + "got_raw": "Language.CHINESE", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "temporalio/samples-python", + "expected": "CHECKSUM", + "got": "RETURNED_PATH", + "got_raw": "RETURNED_PATH", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "temporalio/sdk-python", + "expected": "\"my query\"", + "got": "\"my query\"", + "got_raw": "\"my query\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "temporalio/sdk-python", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "temporalio/sdk-python", + "expected": "\"\")", + "got": "b\"\")", + "got_raw": "b\"\")", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "temporalio/sdk-python", + "expected": "\"inbound\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.26666666666666666 + }, + { + "repo": "temporalio/sdk-python", + "expected": "\"done\"", + "got": "\"shutdown\"", + "got_raw": "\"shutdown\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.625 + }, + { + "repo": "temporalio/sdk-python", + "expected": "str(err.value)", + "got": "str(err.value)", + "got_raw": "str(err.value)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "temporalio/sdk-python", + "expected": "check)", + "got": "check )", + "got_raw": " check )", + "exact_match": false, + "code_bleu": 0.7071067811865476, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "temporalio/sdk-python", + "expected": "expect_equal", + "got": "expect_equal", + "got_raw": "expect_equal", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "temporalio/sdk-python", + "expected": "len(job_protos) - 1", + "got": "len(jobs_with_payloads) + 1) == len(job_protos)", + "got_raw": "len(jobs_with_payloads) + 1) == len(job_protos)", + "exact_match": false, + "code_bleu": 0.27824623288353134, + "edit_similarity": 0.45454545454545453 + }, + { + "repo": "temporalio/sdk-python", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "temporalio/sdk-python", + "expected": "\"bar\"", + "got": "\"bar\"", + "got_raw": "\"bar\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "temporalio/sdk-python", + "expected": "b", + "got": "b", + "got_raw": "b", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "temporalio/sdk-python", + "expected": "err.value.cause)", + "got": "err.value)", + "got_raw": "err.value)", + "exact_match": false, + "code_bleu": 0.4714045207910316, + "edit_similarity": 0.7692307692307693 + }, + { + "repo": "temporalio/sdk-python", + "expected": "\"__repr__\"", + "got": "\"<__str__>\"", + "got_raw": "\"<__str__>\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "temporalio/sdk-python", + "expected": "check)", + "got": "10, check)", + "got_raw": "10, check)", + "exact_match": false, + "code_bleu": 0.47287080450158786, + "edit_similarity": 0.75 + }, + { + "repo": "temporalio/sdk-python", + "expected": "[\"param: param1\", \"task, type: unknown\", \"cancelled=True\"]", + "got": "[\"param1\"]", + "got_raw": "[\"param1\"]", + "exact_match": false, + "code_bleu": 0.21428571428571425, + "edit_similarity": 0.29411764705882354 + }, + { + "repo": "temporalio/sdk-python", + "expected": "\"replaced key\"", + "got": "\"api_key\"", + "got_raw": "\"api_key\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5217391304347826 + }, + { + "repo": "temporalio/sdk-python", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "temporalio/sdk-python", + "expected": "version", + "got": "version", + "got_raw": "version", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "temporalio/sdk-python", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "textstat/textstat", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "textstat/textstat", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "textstat/textstat", + "expected": "margin", + "got": "margin", + "got_raw": "margin", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "textstat/textstat", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "textstat/textstat", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "textstat/textstat", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "textstat/textstat", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tfeldmann/organize", + "expected": "{ \"for-test1.txt\": \"WRITE test1.txt\", \"for-test2.txt\": \"WRITE test2.txt\", }", + "got": "{\"for-test1.txt\": \"WRITE test1.txt\\n\", \"for-test2.txt\": \"WRITE test2.txt\\n\"}", + "got_raw": "{\"for-test1.txt\": \"WRITE test1.txt\\n\", \"for-test2.txt\": \"WRITE test2.txt\\n\"}", + "exact_match": false, + "code_bleu": 0.35353318581046167, + "edit_similarity": 0.9536423841059603 + }, + { + "repo": "tfeldmann/organize", + "expected": "[ \"Odd student numbers: student-01\", \"Odd student numbers: student-03\", ]", + "got": "[\"Odd student numbers: student-01.txt\", \"Odd student numbers: student-03.txt\"]", + "got_raw": "[\"Odd student numbers: student-01.txt\", \"Odd student numbers: student-03.txt\"]", + "exact_match": false, + "code_bleu": 0.2705565962292491, + "edit_similarity": 0.9271523178807947 + }, + { + "repo": "tfeldmann/organize", + "expected": "[ \"Found JPG file: asd\", \"Found JPG file: camel\", \"Found JPG file: test\", ]", + "got": "[\"Found JPG file: test.jpg\", \"Found JPG file: asd.JPG\"]", + "got_raw": "[\"Found JPG file: test.jpg\", \"Found JPG file: asd.JPG\"]", + "exact_match": false, + "code_bleu": 0.20291744717193683, + "edit_similarity": 0.4 + }, + { + "repo": "tfeldmann/organize", + "expected": "FILES", + "got": "FILES", + "got_raw": "FILES", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tfeldmann/organize", + "expected": "[\"file (green)\"]", + "got": "[\"GREEN\"]", + "got_raw": "[\"GREEN\"]", + "exact_match": false, + "code_bleu": 0.49999999999999994, + "edit_similarity": 0.32 + }, + { + "repo": "tfeldmann/organize", + "expected": "{ \"2000\": {\"01\": {\"12\": {\"photo1\": \"\"}}}, \"2020\": {\"01\": {\"01\": {\"photo2\": \"\", \"photo3\": \"\"}}}, }", + "got": "[\"2000/01/12/photo1\", \"2020/01/01/photo2\", \"2020/01/01/photo3\"]", + "got_raw": "[\"2000/01/12/photo1\", \"2020/01/01/photo2\", \"2020/01/01/photo3\"]", + "exact_match": false, + "code_bleu": 0.03786756997767944, + "edit_similarity": 0.55 + }, + { + "repo": "tfeldmann/organize", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tfeldmann/organize", + "expected": "{ \"file.txt\": \"Hello world\\nAnother line\", \"folder\": { \"empty_sub\": {}, }, \"empty_folder\": {}, }", + "got": "{}", + "got_raw": "{}", + "exact_match": false, + "code_bleu": 0.06734350297014738, + "edit_similarity": 0.04081632653061224 + }, + { + "repo": "tfeldmann/organize", + "expected": "{\"testtest.py\": \"\"}", + "got": "[\"test.py\"]", + "got_raw": "[\"test.py\"]", + "exact_match": false, + "code_bleu": 0.22795070569547776, + "edit_similarity": 0.6 + }, + { + "repo": "tfeldmann/organize", + "expected": "Path(result)", + "got": "Path(result)", + "got_raw": "Path(result)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tfeldmann/organize", + "expected": "Counter([\"fileA\", \"fileB\"])", + "got": "{\"fileA.txt\": \"fileA\", \"fileB.txt\": \"fileB\"}", + "got_raw": "{\"fileA.txt\": \"fileA\", \"fileB.txt\": \"fileB\"}", + "exact_match": false, + "code_bleu": 0.19960198807747329, + "edit_similarity": 0.4788732394366197 + }, + { + "repo": "tfeldmann/organize", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tfeldmann/organize", + "expected": "{new_name: \"\"}", + "got": "[\"test.txt\"]", + "got_raw": "[\"test.txt\"]", + "exact_match": false, + "code_bleu": 0.22795070569547776, + "edit_similarity": 0.15384615384615385 + }, + { + "repo": "tfeldmann/organize", + "expected": "{ \"a\": 1, \"b\": {\"b1\": 4, \"b2\": 3}, }", + "got": "a", + "got_raw": "a", + "exact_match": false, + "code_bleu": 0.027777777777777776, + "edit_similarity": 0.05405405405405406 + }, + { + "repo": "theQRL/QRL", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "theQRL/QRL", + "expected": "self.test_data)", + "got": "self.test_data)", + "got_raw": "self.test_data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "theQRL/QRL", + "expected": "8)", + "got": "8)", + "got_raw": "8)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "theQRL/QRL", + "expected": "4)", + "got": "1)", + "got_raw": "1)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "theQRL/QRL", + "expected": "1)", + "got": "StringToUInt256('1'))", + "got_raw": "StringToUInt256('1'))", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.17391304347826086 + }, + { + "repo": "theQRL/QRL", + "expected": "json.loads(txjson))", + "got": "json.loads(txjson))", + "got_raw": "json.loads(txjson))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "theQRL/QRL", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "theQRL/QRL", + "expected": "result)", + "got": "result)", + "got_raw": "result)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "theQRL/QRL", + "expected": "bin2hstr(tx.txhash))", + "got": "bin2hstr(tx.sig))", + "got_raw": "bin2hstr(tx.sig))", + "exact_match": false, + "code_bleu": 0.5946035575013605, + "edit_similarity": 0.8108108108108109 + }, + { + "repo": "theQRL/QRL", + "expected": "tx)", + "got": "tx.verify())", + "got_raw": "tx.verify())", + "exact_match": false, + "code_bleu": 0.24446151121745047, + "edit_similarity": 0.4 + }, + { + "repo": "theQRL/QRL", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "theQRL/QRL", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "theQRL/QRL", + "expected": "output)", + "got": "output)", + "got_raw": "output)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "theQRL/QRL", + "expected": "3)", + "got": "3)", + "got_raw": "3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "theQRL/QRL", + "expected": "2)", + "got": "2)", + "got_raw": "2)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "theQRL/QRL", + "expected": "xmss.hash_function)", + "got": "xmss.hash_function)", + "got_raw": "xmss.hash_function)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "theQRL/QRL", + "expected": "1)", + "got": "1)", + "got_raw": "1)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "theQRL/QRL", + "expected": "block_11b._data)", + "got": "block_11b._data)", + "got_raw": "block_11b._data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "theQRL/QRL", + "expected": "4)", + "got": "5)", + "got_raw": "5)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "theQRL/QRL", + "expected": "tx.ots_key)", + "got": "tx.amounts[0])", + "got_raw": "tx.amounts[0])", + "exact_match": false, + "code_bleu": 0.2626909894424158, + "edit_similarity": 0.56 + }, + { + "repo": "tmbo/questionary", + "expected": "\"t\"", + "got": "\"python3\"", + "got_raw": "\"python3\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "tmbo/questionary", + "expected": "\"write\"", + "got": "\"write\"", + "got_raw": "\"write\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmbo/questionary", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmbo/questionary", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmbo/questionary", + "expected": "filtered", + "got": "filtered", + "got_raw": "filtered", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmbo/questionary", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmbo/questionary", + "expected": "{\"hello\": None}", + "got": "{\"hello\": None}", + "got_raw": "{\"hello\": None}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmbo/questionary", + "expected": "KeyboardInterrupt)", + "got": "KeyboardInterrupt)", + "got_raw": "KeyboardInterrupt)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmbo/questionary", + "expected": "{\"theme\": \"Ask opening hours\"}", + "got": "{\"a\": 1, \"b\": 2, \"c\": 3}", + "got_raw": "{\"a\": 1, \"b\": 2, \"c\": 3}", + "exact_match": false, + "code_bleu": 0.1135935489027116, + "edit_similarity": 0.2962962962962963 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "data", + "got": "data", + "got_raw": "data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "exc.WorkspaceError)", + "got": "exc.TmuxpException)", + "got_raw": "exc.TmuxpException)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.47368421052631576 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "\"test\"", + "got": "\"test\"", + "got_raw": "\"test\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "output", + "got": "output", + "got_raw": "output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "\"valid_input\"", + "got": "\"valid_input\"", + "got_raw": "\"valid_input\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "cli_output", + "got": "cli_output", + "got_raw": "cli_output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "masked", + "got": "masked", + "got_raw": "masked", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "mock", + "got": "mock", + "got_raw": "mock", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "prompt", + "got": "prompt", + "got_raw": "prompt", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "sconfig[\"start_directory\"]", + "got": "sconfig.start_directory", + "got_raw": "sconfig.start_directory", + "exact_match": false, + "code_bleu": 0.3388507513536918, + "edit_similarity": 0.8979591836734694 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tmux-python/tmuxp", + "expected": "config_fixture.expand1.after_workspace()", + "got": "config_fixture.expand1.after_workspace", + "got_raw": "config_fixture.expand1.after_workspace", + "exact_match": false, + "code_bleu": 0.7142857142857143, + "edit_similarity": 0.9743589743589743 + }, + { + "repo": "tomerfiliba/plumbum", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tomerfiliba/plumbum", + "expected": "\"\\033[1m\"", + "got": "\"\\033[1m\"", + "got_raw": "\"\\033[1m\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tomerfiliba/plumbum", + "expected": "rem", + "got": "rem", + "got_raw": "rem", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tomerfiliba/plumbum", + "expected": "wrapped", + "got": "wrapped", + "got_raw": "wrapped", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tomerfiliba/plumbum", + "expected": "output", + "got": "output", + "got_raw": "output", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tomerfiliba/plumbum", + "expected": "\"lala\"", + "got": "\"lala\"", + "got_raw": "\"lala\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tomerfiliba/plumbum", + "expected": "colors[2]", + "got": "colors.red", + "got_raw": "colors.red", + "exact_match": false, + "code_bleu": 0.3388507513536918, + "edit_similarity": 0.631578947368421 + }, + { + "repo": "tomerfiliba/plumbum", + "expected": "data", + "got": "data", + "got_raw": "data", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tomerfiliba/plumbum", + "expected": "\"This is output\\n\" * 2", + "got": "\"This is output\\nThis is output\\n\"", + "got_raw": "\"This is output\\nThis is output\\n\"", + "exact_match": false, + "code_bleu": 0.16666666666666666, + "edit_similarity": 0.6785714285714286 + }, + { + "repo": "tomerfiliba/plumbum", + "expected": "13", + "got": "13", + "got_raw": "13", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tomerfiliba/plumbum", + "expected": "8", + "got": "8", + "got_raw": "8", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tomerfiliba/plumbum", + "expected": "10000", + "got": "5000", + "got_raw": "5000", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "tomerfiliba/plumbum", + "expected": "Color(\"Dark_Blue\")", + "got": "Color(\"dark blue\")", + "got_raw": "Color(\"dark blue\")", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8333333333333334 + }, + { + "repo": "tomerfiliba/plumbum", + "expected": "\"hi\"", + "got": "\"hi\"", + "got_raw": "\"hi\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "tomerfiliba/plumbum", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ultrabug/mkdocs-static-i18n", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ultrabug/mkdocs-static-i18n", + "expected": "[ {\"name\": \"english\", \"link\": \"/\", \"lang\": \"en\"}, {\"name\": \"fran\u00e7ais\", \"link\": \"/fr/\", \"lang\": \"fr\"}, ]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.03964023716675737, + "edit_similarity": 0.0380952380952381 + }, + { + "repo": "ultrabug/mkdocs-static-i18n", + "expected": "[ { \"locale\": \"en\", \"name\": \"english\", \"link\": \"/\", \"fixed_link\": None, \"build\": True, \"default\": True, \"nav\": None, \"nav_translations\": None, \"admonition_translations\": None, \"theme\": None, \"site_name\": None, \"copyright\": None, \"extra\": None, \"site_author\": None, \"site_description\": None, \"site_url\": None, }, ]", + "got": "[\"en\"]", + "got_raw": "[\"en\"]", + "exact_match": false, + "code_bleu": 0.023360215909207435, + "edit_similarity": 0.012539184952978056 + }, + { + "repo": "ultrabug/mkdocs-static-i18n", + "expected": "test_page.dest_uri", + "got": "test_page.dest_uri", + "got_raw": "test_page.dest_uri", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "ultrabug/mkdocs-static-i18n", + "expected": "len(nav.pages) + 1", + "got": "len(nav)", + "got_raw": "len(nav)", + "exact_match": false, + "code_bleu": 0.35355339059327373, + "edit_similarity": 0.6153846153846154 + }, + { + "repo": "ultrabug/mkdocs-static-i18n", + "expected": "\"en\"", + "got": "\"en\"", + "got_raw": "\"en\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unfoldadmin/django-unfold", + "expected": "False", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "unfoldadmin/django-unfold", + "expected": "{ \"light\": static(\"icon-light.svg\"), \"dark\": static(\"icon-dark.svg\"), }", + "got": "static(\"icon-light.svg\")", + "got_raw": "static(\"icon-light.svg\")", + "exact_match": false, + "code_bleu": 0.25, + "edit_similarity": 0.5052631578947369 + }, + { + "repo": "unfoldadmin/django-unfold", + "expected": "response.content.decode()", + "got": "response.content.decode()", + "got_raw": "response.content.decode()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unfoldadmin/django-unfold", + "expected": "\"/lambda/link\"", + "got": "lambda request: \"/lambda/link\")", + "got_raw": "lambda request: \"/lambda/link\")", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.6222222222222222 + }, + { + "repo": "unfoldadmin/django-unfold", + "expected": "\"Custom readonly field\"", + "got": "\"Custom readonly field\"", + "got_raw": "\"Custom readonly field\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unfoldadmin/django-unfold", + "expected": "rendered", + "got": "rendered", + "got_raw": "rendered", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unfoldadmin/django-unfold", + "expected": "HTTPStatus.OK", + "got": "HTTPStatus.OK", + "got_raw": "HTTPStatus.OK", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unionai-oss/pandera", + "expected": "str(e.value)", + "got": "str(e.value)", + "got_raw": "str(e.value)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unionai-oss/pandera", + "expected": "buffer", + "got": "buffer", + "got_raw": "buffer", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unionai-oss/pandera", + "expected": "csv_data)", + "got": "csv_data)", + "got_raw": "csv_data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unionai-oss/pandera", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unionai-oss/pandera", + "expected": "same_check", + "got": "same_check", + "got_raw": "same_check", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unionai-oss/pandera", + "expected": "[\"1\", \"2\", \"c\"]", + "got": "[2, 3]", + "got_raw": "[2, 3]", + "exact_match": false, + "code_bleu": 0.23190565391078496, + "edit_similarity": 0.38095238095238093 + }, + { + "repo": "unionai-oss/pandera", + "expected": "to_dtype.type", + "got": "to_dtype", + "got_raw": "to_dtype", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.7619047619047619 + }, + { + "repo": "unionai-oss/pandera", + "expected": "pa.errors.SchemaError)", + "got": "pa.errors.SchemaError)", + "got_raw": "pa.errors.SchemaError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unionai-oss/pandera", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unionai-oss/pandera", + "expected": "expected_dtype.scale", + "got": "expected_dtype.scale", + "got_raw": "expected_dtype.scale", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unionai-oss/pandera", + "expected": "json_str)", + "got": "json_str)", + "got_raw": "json_str)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unionai-oss/pandera", + "expected": "str(exc_native.data)", + "got": "exc_native.data", + "got_raw": "exc_native.data", + "exact_match": false, + "code_bleu": 0.42044820762685725, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "unionai-oss/pandera", + "expected": "\"a\"", + "got": "Base.a", + "got_raw": "Base.a", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "unionai-oss/pandera", + "expected": "len(example)", + "got": "len(example)", + "got_raw": "len(example)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unionai-oss/pandera", + "expected": "expected.type", + "got": "expected.type", + "got_raw": "expected.type", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unionai-oss/pandera", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unionai-oss/pandera", + "expected": "pd.StringDtype()", + "got": "pd.StringDtype()", + "got_raw": "pd.StringDtype()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unionai-oss/pandera", + "expected": "buffer)", + "got": "param=\"value\")", + "got_raw": "param=\"value\")", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "unionai-oss/pandera", + "expected": "ibis.__version__", + "got": "ibis.__version__", + "got_raw": "ibis.__version__", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "unionai-oss/pandera", + "expected": "value", + "got": "value", + "got_raw": "value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "uriyyo/fastapi-pagination", + "expected": "TypeError, match=r\"^CursorPage should be used with CursorParams$\")", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.16988673071467442, + "edit_similarity": 0.2631578947368421 + }, + { + "repo": "uriyyo/fastapi-pagination", + "expected": "int", + "got": "int", + "got_raw": "int", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "uriyyo/fastapi-pagination", + "expected": "ValueError, match=r\"^Not a 'cursor' params$\")", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": false, + "code_bleu": 0.153509990260506, + "edit_similarity": 0.2545454545454545 + }, + { + "repo": "uriyyo/fastapi-pagination", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "uriyyo/fastapi-pagination", + "expected": "response()", + "got": "response(req)", + "got_raw": "response(req)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.8695652173913043 + }, + { + "repo": "uriyyo/fastapi-pagination", + "expected": "str", + "got": "str", + "got_raw": "str", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "uriyyo/fastapi-pagination", + "expected": "3", + "got": "3", + "got_raw": "3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "urllib3/urllib3", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "urllib3/urllib3", + "expected": "bytes", + "got": "bytes", + "got_raw": "bytes", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "urllib3/urllib3", + "expected": "EmptyPoolError)", + "got": "EmptyPoolError)", + "got_raw": "EmptyPoolError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "urllib3/urllib3", + "expected": "3", + "got": "4", + "got_raw": "4", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "urllib3/urllib3", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "urllib3/urllib3", + "expected": "b\"\"", + "got": "b\"\"", + "got_raw": "b\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "urllib3/urllib3", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "urllib3/urllib3", + "expected": "443", + "got": "443", + "got_raw": "443", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "urllib3/urllib3", + "expected": "\"http://example.com/\"", + "got": "\"http://localhost:8080/\"", + "got_raw": "\"http://localhost:8080/\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5333333333333333 + }, + { + "repo": "urllib3/urllib3", + "expected": "80", + "got": "80", + "got_raw": "80", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "urllib3/urllib3", + "expected": "ResponseNotReady)", + "got": "ResponseNotReady)", + "got_raw": "ResponseNotReady)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "urllib3/urllib3", + "expected": "5", + "got": "7", + "got_raw": "7", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "urllib3/urllib3", + "expected": "mock.call(ciphers)", + "got": "mock.call(ciphers)", + "got_raw": "mock.call(ciphers)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "urllib3/urllib3", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "urllib3/urllib3", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "urllib3/urllib3", + "expected": "default_headers", + "got": "default_headers", + "got_raw": "default_headers", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "urllib3/urllib3", + "expected": "0", + "got": "0.2", + "got_raw": "0.2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "urllib3/urllib3", + "expected": "OSError)", + "got": "ConnectionError)", + "got_raw": "ConnectionError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.5 + }, + { + "repo": "urllib3/urllib3", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "urllib3/urllib3", + "expected": "msg", + "got": "msg", + "got_raw": "msg", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vkbottle/vkbottle", + "expected": "expected_error", + "got": "expected_error", + "got_raw": "expected_error", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vkbottle/vkbottle", + "expected": "{ \"peer_ids\": str(message.peer_id), \"random_id\": 0, \"some_unsigned_param\": \"test\", \"r\": 1, }", + "got": "{", + "got_raw": "{\n \"peer_ids\": str(message.peer_id),\n \"r\": 1,\n \"random_id\": 0,\n \"some_unsigned_param\": \"test\",\n }", + "exact_match": false, + "code_bleu": 0.025852328587015673, + "edit_similarity": 0.021505376344086023 + }, + { + "repo": "vkbottle/vkbottle", + "expected": "msg", + "got": "msg", + "got_raw": "msg", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vkbottle/vkbottle", + "expected": "{ \"version\": Formatter.VERSION, \"items\": [ {\"type\": \"bold\", \"offset\": 0, \"length\": 7}, {\"type\": \"italic\", \"offset\": 7, \"length\": 6}, ], }", + "got": "{\"text\": \"Hello, World!\"}", + "got_raw": "{\"text\": \"Hello, World!\"}", + "exact_match": false, + "code_bleu": 0.0395936482286706, + "edit_similarity": 0.16568047337278108 + }, + { + "repo": "vkbottle/vkbottle", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vkbottle/vkbottle", + "expected": "False", + "got": "False", + "got_raw": "False", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vkbottle/vkbottle", + "expected": "bot.router.views[\"raw\"]", + "got": "bot.router.views[\"raw_event\"]", + "got_raw": "bot.router.views[\"raw_event\"]", + "exact_match": false, + "code_bleu": 0.7506238537503395, + "edit_similarity": 0.8846153846153846 + }, + { + "repo": "vkbottle/vkbottle", + "expected": "CodeError42", + "got": "CodeError42", + "got_raw": "CodeError42", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vkbottle/vkbottle", + "expected": "100", + "got": "100", + "got_raw": "100", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vkbottle/vkbottle", + "expected": "expected_exception", + "got": "expected_exception", + "got_raw": "expected_exception", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vkbottle/vkbottle", + "expected": "\"239633676097\"", + "got": "\"239633676097\"", + "got_raw": "\"239633676097\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "\"0\"", + "got": "\"0\"", + "got_raw": "\"0\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "True", + "got": "False", + "got_raw": "False", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "\"root\"", + "got": "\"root\"", + "got_raw": "\"root\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "plot3", + "got": "plot3", + "got_raw": "plot3", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "\"docker\"", + "got": "\"SandboxBackend.DOCKER\"", + "got_raw": "\"SandboxBackend.DOCKER\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.45161290322580644 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "\"cpp\"", + "got": "\"cpp\"", + "got_raw": "\"cpp\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "env_name", + "got": "env_name", + "got_raw": "env_name", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "\"R\"", + "got": "\".R\"", + "got_raw": "\".R\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "\"d3js\"", + "got": "\"d3js\"", + "got_raw": "\"d3js\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "\"js\"", + "got": "\".js\"", + "got_raw": "\".js\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "\"java\"", + "got": "\".java\"", + "got_raw": "\".java\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9230769230769231 + }, + { + "repo": "vndee/llm-sandbox", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "vstorm-co/full-stack-ai-agent-template", + "expected": "60", + "got": "60", + "got_raw": "60", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vstorm-co/full-stack-ai-agent-template", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vstorm-co/full-stack-ai-agent-template", + "expected": "100.0", + "got": "1.0", + "got_raw": "1.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.75 + }, + { + "repo": "vstorm-co/full-stack-ai-agent-template", + "expected": "commands[0][1]", + "got": "commands[0][1]", + "got_raw": "commands[0][1]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vstorm-co/full-stack-ai-agent-template", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vstorm-co/pydantic-deepagents", + "expected": "prompt", + "got": "prompt", + "got_raw": "prompt", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vstorm-co/pydantic-deepagents", + "expected": "b", + "got": "b", + "got_raw": "b", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vstorm-co/pydantic-deepagents", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vstorm-co/pydantic-deepagents", + "expected": "None", + "got": "None", + "got_raw": "None", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vstorm-co/pydantic-deepagents", + "expected": "styles", + "got": "styles", + "got_raw": "styles", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vstorm-co/pydantic-deepagents", + "expected": "\"ok\"", + "got": "\"ok\"", + "got_raw": "\"ok\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vstorm-co/pydantic-deepagents", + "expected": "\"\"", + "got": "\"\"", + "got_raw": "\"\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vstorm-co/pydantic-deepagents", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vstorm-co/pydantic-deepagents", + "expected": "200_000", + "got": "200_000", + "got_raw": "200_000", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vstorm-co/pydantic-deepagents", + "expected": "10", + "got": "10", + "got_raw": "10", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vstorm-co/pydantic-deepagents", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vstorm-co/pydantic-deepagents", + "expected": "50", + "got": "50", + "got_raw": "50", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "vstorm-co/pydantic-deepagents", + "expected": "5", + "got": "5", + "got_raw": "5", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wbopan/moffee", + "expected": "updated_doc", + "got": "updated_doc", + "got_raw": "updated_doc", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wbopan/moffee", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "wbopan/moffee", + "expected": "output_html", + "got": "[\"image.png\", \"image2.png\"]", + "got_raw": "[\"image.png\", \"image2.png\"]", + "exact_match": false, + "code_bleu": 0.2295748846661433, + "edit_similarity": 0.05263157894736842 + }, + { + "repo": "wbopan/moffee", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wbopan/moffee", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webis-de/small-text", + "expected": "clf.num_classes)", + "got": "clf.num_classes)", + "got_raw": "clf.num_classes)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webis-de/small-text", + "expected": "counts[2])", + "got": "counts[2])", + "got_raw": "counts[2])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webis-de/small-text", + "expected": "loss.shape[0])", + "got": "loss.shape[0])", + "got_raw": "loss.shape[0])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webis-de/small-text", + "expected": "y_pred.shape[0])", + "got": "y_pred.shape[0])", + "got_raw": "y_pred.shape[0])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webis-de/small-text", + "expected": "proba.shape[0])", + "got": "proba.shape[0])", + "got_raw": "proba.shape[0])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webis-de/small-text", + "expected": "stop)", + "got": "stop)", + "got_raw": "stop)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webis-de/small-text", + "expected": "len(ds))", + "got": "len(ds))", + "got_raw": "len(ds))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webis-de/small-text", + "expected": "output.size(0))", + "got": "output.shape[0])", + "got_raw": "output.shape[0])", + "exact_match": false, + "code_bleu": 0.2777619034011791, + "edit_similarity": 0.7096774193548387 + }, + { + "repo": "webis-de/small-text", + "expected": "model)", + "got": "model)", + "got_raw": "model)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webis-de/small-text", + "expected": "len(dataset))", + "got": "len(dataset))", + "got_raw": "len(dataset))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webis-de/small-text", + "expected": "proba.shape[0])", + "got": "proba.shape[0])", + "got_raw": "proba.shape[0])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webis-de/small-text", + "expected": "amp_args.use_amp)", + "got": "amp_args.use_amp)", + "got_raw": "amp_args.use_amp)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webis-de/small-text", + "expected": "str(query_strategy))", + "got": "str(query_strategy))", + "got_raw": "str(query_strategy))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webis-de/small-text", + "expected": "model.dropout.p)", + "got": "model.dropout)", + "got_raw": "model.dropout)", + "exact_match": false, + "code_bleu": 0.4714045207910316, + "edit_similarity": 0.9333333333333333 + }, + { + "repo": "webis-de/small-text", + "expected": "y_pred_proba.shape)", + "got": "y_pred_proba.shape)", + "got_raw": "y_pred_proba.shape)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webis-de/small-text", + "expected": "strategy.pbar)", + "got": "strategy.progress_bar)", + "got_raw": "strategy.progress_bar)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7777777777777778 + }, + { + "repo": "webis-de/small-text", + "expected": "amp_args.use_amp)", + "got": "amp_args.enabled)", + "got_raw": "amp_args.enabled)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.7058823529411765 + }, + { + "repo": "webis-de/small-text", + "expected": "clf.device)", + "got": "clf.device)", + "got_raw": "clf.device)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webis-de/small-text", + "expected": "len(ds.x))", + "got": "len(ds))", + "got_raw": "len(ds))", + "exact_match": false, + "code_bleu": 0.4316250567652526, + "edit_similarity": 0.8888888888888888 + }, + { + "repo": "webis-de/small-text", + "expected": "result.x)", + "got": "result.x)", + "got_raw": "result.x)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webisopen/OpenAgent", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webisopen/OpenAgent", + "expected": "event_data_input_", + "got": "event_data_input_", + "got_raw": "event_data_input_", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webisopen/OpenAgent", + "expected": "\"TransferExecutor\"", + "got": "\"TransferAsset\"", + "got_raw": "\"TransferAsset\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.7272727272727273 + }, + { + "repo": "webisopen/OpenAgent", + "expected": "\"binance\"", + "got": "\"binance\"", + "got_raw": "\"binance\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "webisopen/OpenAgent", + "expected": "\"ETH\"", + "got": "\"eth\"", + "got_raw": "\"eth\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "wee-slack/wee-slack", + "expected": "\"conversations.setTopic\"", + "got": "\"topic\"", + "got_raw": "\"topic\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.45161290322580644 + }, + { + "repo": "wee-slack/wee-slack", + "expected": "ts_different_major", + "got": "ts_different_major", + "got_raw": "ts_different_major", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wee-slack/wee-slack", + "expected": "(command, 0, \"o1o2o3\", \"e1e2e3\")", + "got": "(command, 0, \"o3\", \"e3\")", + "got_raw": "(command, 0, \"o3\", \"e3\")", + "exact_match": false, + "code_bleu": 0.5555238068023582, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "wee-slack/wee-slack", + "expected": "deleted_str", + "got": "deleted_str in deleted_message_channel.messages[deleted_message_ts].message_json[\"text\"]", + "got_raw": "deleted_str in deleted_message_channel.messages[deleted_message_ts].message_json[\"text\"]", + "exact_match": true, + "code_bleu": 0.09552040806823771, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "wee-slack/wee-slack", + "expected": "case[\"rendered\"]", + "got": "case[\"rendered\"]", + "got_raw": "case[\"rendered\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wee-slack/wee-slack", + "expected": "\"response\"", + "got": "(\"response\",)", + "got_raw": "(\"response\",)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.8695652173913043 + }, + { + "repo": "wee-slack/wee-slack", + "expected": "type(encode_to_utf8(value_set))", + "got": "type(value_tuple)", + "got_raw": "type(value_tuple)", + "exact_match": false, + "code_bleu": 0.29036157037406557, + "edit_similarity": 0.5416666666666666 + }, + { + "repo": "wee-slack/wee-slack", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wee-slack/wee-slack", + "expected": "e", + "got": "e", + "got_raw": "e", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wee-slack/wee-slack", + "expected": "\"away\"", + "got": "\"away\"", + "got_raw": "\"away\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wee-slack/wee-slack", + "expected": "subteam.description", + "got": "subteam.description", + "got_raw": "subteam.description", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wee-slack/wee-slack", + "expected": "\": my test message\"", + "got": "\"<", + "got_raw": "\"<#C1234567890> my test message\"", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.12903225806451613 + }, + { + "repo": "wee-slack/wee-slack", + "expected": "\"generally, yep!\"", + "got": "\"Hello, world!\"", + "got_raw": "\"Hello, world!\"", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "wee-slack/wee-slack", + "expected": "value", + "got": "value", + "got_raw": "value", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "option_values.max_awaits)", + "got": "option_values.max_awaits)", + "got_raw": "option_values.max_awaits)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "[])", + "got": "[])", + "got_raw": "[])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "wemake-services/wemake-python-styleguide", + "expected": "bad_name)", + "got": "bad_name)", + "got_raw": "bad_name)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "whyisdifficult/jiratui", + "expected": "expected_result", + "got": "expected_result", + "got_raw": "expected_result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "whyisdifficult/jiratui", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "whyisdifficult/jiratui", + "expected": "[ RelatedJiraIssue( id='10075', key='SCRUM-1', summary='(Sample) Payment Processing with Async', status=IssueStatus( name='In Progress', id='10001', ), issue_type=IssueType(id='10001', name='Epic', scope_project=None), link_type='is cloned by', relation_type='inward', priority=IssuePriority(id='3', name='Medium'), ), ]", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.016988673071467442, + "edit_similarity": 0.012422360248447204 + }, + { + "repo": "whyisdifficult/jiratui", + "expected": "'Default Title'", + "got": "'Default Title'", + "got_raw": "'Default Title'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "whyisdifficult/jiratui", + "expected": "'abd'", + "got": "'abcd'", + "got_raw": "'abcd'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.9090909090909091 + }, + { + "repo": "whyisdifficult/jiratui", + "expected": "'dracula'", + "got": "'flexoki'", + "got_raw": "'flexoki'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.3333333333333333 + }, + { + "repo": "whyisdifficult/jiratui", + "expected": "'3')", + "got": "attachment_dt._selected_attachment_id)", + "got_raw": "attachment_dt._selected_attachment_id)", + "exact_match": false, + "code_bleu": 0.35930411196308426, + "edit_similarity": 0.047619047619047616 + }, + { + "repo": "whyisdifficult/jiratui", + "expected": "datetime(2025, 7, 5, 14, 34, 59)", + "got": "datetime(2025, 12, 30, 18, 0)", + "got_raw": "datetime(2025, 12, 30, 18, 0)", + "exact_match": false, + "code_bleu": 0.28710273340265535, + "edit_similarity": 0.7213114754098361 + }, + { + "repo": "whyisdifficult/jiratui", + "expected": "'Hello World!'", + "got": "'Hello World!'", + "got_raw": "'Hello World!'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "whyisdifficult/jiratui", + "expected": "'project = \"P1\" and created >= \"2025-12-01\" and created <= \"2025-12-02\" and updated >= \"2025-12-03\" and updated <= \"2025-12-04\" and status = \"1\" and assignee = \"1\" and type = 2 and q=5 order by key asc'", + "got": "'project = P1 AND created >= 2025-12-01 AND created <= 2025-12-02 AND updated >= 2025-12-03 AND updated <= 2025-12-04 AND status = 1 AND assignee = 1 AND issue_type = 2 AND q = 5 ORDER BY key ASC'", + "got_raw": "'project = P1 AND created >= 2025-12-01 AND created <= 2025-12-02 AND updated >= 2025-12-03 AND updated <= 2025-12-04 AND status = 1 AND assignee = 1 AND issue_type = 2 AND q = 5 ORDER BY key ASC'", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.6030150753768844 + }, + { + "repo": "whyisdifficult/jiratui", + "expected": "'key-2'", + "got": "'key-2'", + "got_raw": "'key-2'", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "whyisdifficult/jiratui", + "expected": "expected_exception, match='some error message')", + "got": "expected_exception)", + "got_raw": "expected_exception)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.5757575757575758 + }, + { + "repo": "worldcoin/open-iris", + "expected": "normalise", + "got": "normalise", + "got_raw": "normalise", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "worldcoin/open-iris", + "expected": "result.pupil_array)", + "got": "result.pupil_array)", + "got_raw": "result.pupil_array)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "worldcoin/open-iris", + "expected": "metadata_1[\"occlusion90\"])", + "got": "metadata_2[\"eye_centers\"][\"pupil_center\"])", + "got_raw": "metadata_2[\"eye_centers\"][\"pupil_center\"])", + "exact_match": false, + "code_bleu": 0.22679164443904004, + "edit_similarity": 0.47058823529411764 + }, + { + "repo": "worldcoin/open-iris", + "expected": "result)", + "got": "result)", + "got_raw": "result)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "worldcoin/open-iris", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "worldcoin/open-iris", + "expected": "\"v0.1\"", + "got": "expected_result.iris_code_version", + "got_raw": "expected_result.iris_code_version", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.05128205128205128 + }, + { + "repo": "worldcoin/open-iris", + "expected": "mock_metadata", + "got": "mock_metadata", + "got_raw": "mock_metadata", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "worldcoin/open-iris", + "expected": "np.sort(result_ys))", + "got": "np.sort(result_ys))", + "got_raw": "np.sort(result_ys))", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "worldcoin/open-iris", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "worldcoin/open-iris", + "expected": "ValidationError)", + "got": "ValidationError)", + "got_raw": "ValidationError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "worldcoin/open-iris", + "expected": "frame", + "got": "frame", + "got_raw": "frame", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "worldcoin/open-iris", + "expected": "__version__", + "got": "__version__", + "got_raw": "__version__", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "worldcoin/open-iris", + "expected": "6", + "got": "6", + "got_raw": "6", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "worldcoin/open-iris", + "expected": "plain_str", + "got": "plain_str", + "got_raw": "plain_str", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "worldcoin/open-iris", + "expected": "error_message", + "got": "error_message", + "got_raw": "error_message", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "worldcoin/open-iris", + "expected": "mock_model.params.ret_msg", + "got": "\"Success\"", + "got_raw": "\"Success\"", + "exact_match": false, + "code_bleu": 0.1, + "edit_similarity": 0.23529411764705882 + }, + { + "repo": "worldcoin/open-iris", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "worldcoin/open-iris", + "expected": "error_dict_2 is None", + "got": "error_dict_2 is None)", + "got_raw": "error_dict_2 is None)", + "exact_match": false, + "code_bleu": 0.668740304976422, + "edit_similarity": 0.975609756097561 + }, + { + "repo": "worldcoin/open-iris", + "expected": "expected_index", + "got": "expected_index", + "got_raw": "expected_index", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "worldcoin/open-iris", + "expected": "result.iris_array)", + "got": "result.iris_array)", + "got_raw": "result.iris_array)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xavctn/img2table", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xavctn/img2table", + "expected": "(h_lines_expected, v_lines_expected)", + "got": "(h_lines_expected, v_lines_expected)", + "got_raw": "(h_lines_expected, v_lines_expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xavctn/img2table", + "expected": "51", + "got": "1.0", + "got_raw": "1.0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.4 + }, + { + "repo": "xavctn/img2table", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xavctn/img2table", + "expected": "expected", + "got": "expected", + "got_raw": "expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xavctn/img2table", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xavctn/img2table", + "expected": "table.nb_rows + 20", + "got": "20", + "got_raw": "20", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.2 + }, + { + "repo": "xavctn/img2table", + "expected": "40", + "got": "40", + "got_raw": "40", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xavctn/img2table", + "expected": "16", + "got": "3", + "got_raw": "3", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "xavctn/img2table", + "expected": "EnvironmentError)", + "got": "ModuleNotFoundError)", + "got_raw": "ModuleNotFoundError)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.4864864864864865 + }, + { + "repo": "xavctn/img2table", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xavctn/img2table", + "expected": "len(ocr_df.df)", + "got": "len(ocr_df.df)", + "got_raw": "len(ocr_df.df)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xavctn/img2table", + "expected": "set(expected)", + "got": "set(expected)", + "got_raw": "set(expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xavctn/img2table", + "expected": "(417, 1365, 3)", + "got": "img_from_bytes.images[0].shape", + "got_raw": "img_from_bytes.images[0].shape", + "exact_match": false, + "code_bleu": 0.13485111859503685, + "edit_similarity": 0.0 + }, + { + "repo": "xaviml/controllerx", + "expected": "expected_direction", + "got": "expected_direction", + "got_raw": "expected_direction", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xaviml/controllerx", + "expected": "\"media_player/media_previous_track\")", + "got": "(\"media_player\", \"media_previous_track\")", + "got_raw": "(\"media_player\", \"media_previous_track\")", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.9210526315789473 + }, + { + "repo": "xaviml/controllerx", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xaviml/controllerx", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xaviml/controllerx", + "expected": "stepper)", + "got": "stepper)", + "got_raw": "stepper)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xaviml/controllerx", + "expected": "\"homematic.keypress\")", + "got": "\"test_event\")", + "got_raw": "\"test_event\")", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.35294117647058826 + }, + { + "repo": "xaviml/controllerx", + "expected": "expected)", + "got": "expected)", + "got_raw": "expected)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xaviml/controllerx", + "expected": "args[ENTITY_ARG].get( \"attr_test\", DEFAULT_ATTR_TEST )", + "got": "args[ENTITY_ARG][\"attr_test\"]", + "got_raw": "args[ENTITY_ARG][\"attr_test\"]", + "exact_match": false, + "code_bleu": 0.3443250803070525, + "edit_similarity": 0.6506024096385542 + }, + { + "repo": "xaviml/controllerx", + "expected": "\"lutron_caseta_button_event\")", + "got": "(\"lutron_caseta_controller_id\")", + "got_raw": "(\"lutron_caseta_controller_id\")", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.6666666666666666 + }, + { + "repo": "xaviml/controllerx", + "expected": "integration_name_expected", + "got": "integration_name_expected", + "got_raw": "integration_name_expected", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xaviml/controllerx", + "expected": "\"controller_id\")", + "got": "_listen_state_mock.assert_called_once()", + "got_raw": "_listen_state_mock.assert_called_once()", + "exact_match": false, + "code_bleu": 0.2730120862709067, + "edit_similarity": 0.2545454545454545 + }, + { + "repo": "xaviml/controllerx", + "expected": "ValueError)", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xflr6/graphviz", + "expected": "['open', 'nonfilepath'])", + "got": "['open', 'nonfilepath'])", + "got_raw": "['open', 'nonfilepath'])", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xflr6/graphviz", + "expected": "mock_err", + "got": "mock_err", + "got_raw": "mock_err", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xflr6/graphviz", + "expected": "None", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.25 + }, + { + "repo": "xflr6/graphviz", + "expected": "stat_before.st_mtime", + "got": "stat_before.st_mtime", + "got_raw": "stat_before.st_mtime", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xflr6/graphviz", + "expected": "(errno.EACCES, errno.EINVAL)", + "got": "[errno.ENOENT, errno.EACCES]", + "got_raw": "[errno.ENOENT, errno.EACCES]", + "exact_match": false, + "code_bleu": 0.37991784282579627, + "edit_similarity": 0.42857142857142855 + }, + { + "repo": "xflr6/graphviz", + "expected": "expected_exception, match=match)", + "got": "expected_exception)", + "got_raw": "expected_exception)", + "exact_match": false, + "code_bleu": 0.19820118583378682, + "edit_similarity": 0.7450980392156863 + }, + { + "repo": "xflr6/graphviz", + "expected": "expected_quoted", + "got": "expected_quoted", + "got_raw": "expected_quoted", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xflr6/graphviz", + "expected": "('', '' if quiet else stderr)", + "got": "('', stderr)", + "got_raw": "('', stderr)", + "exact_match": false, + "code_bleu": 0.33570837748408533, + "edit_similarity": 0.5853658536585366 + }, + { + "repo": "xflr6/graphviz", + "expected": "'test-output\\\\hello.gv'", + "got": "SOURCE['filename']", + "got_raw": "SOURCE['filename']", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.14634146341463414 + }, + { + "repo": "xflr6/graphviz", + "expected": "str(e.value)", + "got": "str(e.value)", + "got_raw": "str(e.value)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xflr6/graphviz", + "expected": "ValueError, match=r'missing')", + "got": "ValueError)", + "got_raw": "ValueError)", + "exact_match": false, + "code_bleu": 0.16988673071467442, + "edit_similarity": 0.55 + }, + { + "repo": "xflr6/graphviz", + "expected": "OSError)", + "got": "OSError)", + "got_raw": "OSError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xflr6/graphviz", + "expected": "2", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "xflr6/graphviz", + "expected": "_common.INVALID_CMD", + "got": "_common.INVALID_CMD", + "got_raw": "_common.INVALID_CMD", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "xflr6/graphviz", + "expected": "formatter", + "got": "formatter", + "got_raw": "formatter", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "yezz123/authx", + "expected": "\"TEST\"", + "got": "\"TEST\"", + "got_raw": "\"TEST\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "yezz123/authx", + "expected": "\"ERROR\"", + "got": "\"ERROR\"", + "got_raw": "\"ERROR\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "yezz123/authx", + "expected": "401", + "got": "401", + "got_raw": "401", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "yezz123/authx", + "expected": "{}", + "got": "{\"extra\": \"EXTRA\"}", + "got_raw": "{\"extra\": \"EXTRA\"}", + "exact_match": false, + "code_bleu": 0.3021375397356768, + "edit_similarity": 0.2 + }, + { + "repo": "yezz123/authx", + "expected": "exc.AccessTokenRequiredError)", + "got": "exc.RefreshTokenExpired)", + "got_raw": "exc.RefreshTokenExpired)", + "exact_match": false, + "code_bleu": 0.5081327481546147, + "edit_similarity": 0.6037735849056604 + }, + { + "repo": "yezz123/authx", + "expected": "{ \"message\": \"Sample Message\", \"error_type\": ValueError.__name__, }", + "got": "{\"message\": \"Sample Message\"}", + "got_raw": "{\"message\": \"Sample Message\"}", + "exact_match": false, + "code_bleu": 0.3313628036529378, + "edit_similarity": 0.6041666666666666 + }, + { + "repo": "yezz123/authx", + "expected": "JWTDecodeError)", + "got": "JWTDecodeError)", + "got_raw": "JWTDecodeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "yezz123/authx", + "expected": "\"access_token\"", + "got": "\"access_token\"", + "got_raw": "\"access_token\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "yezz123/authx", + "expected": "{\"message\": \"post\"}", + "got": "{\"message\": \"post\"}", + "got_raw": "{\"message\": \"post\"}", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "yezz123/authx", + "expected": "False", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "yezz123/authx", + "expected": "valid_payload.sub", + "got": "valid_payload.sub", + "got_raw": "valid_payload.sub", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "yezz123/authx", + "expected": "\"access\"", + "got": "\"access\"", + "got_raw": "\"access\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zappa/Zappa", + "expected": "\"y\")", + "got": "\"y\")", + "got_raw": "\"y\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zappa/Zappa", + "expected": "url)", + "got": "url)", + "got_raw": "url)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zappa/Zappa", + "expected": "200)", + "got": "200)", + "got_raw": "200)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zappa/Zappa", + "expected": "3)", + "got": "3)", + "got_raw": "3)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zappa/Zappa", + "expected": "\"q\")", + "got": "\"q\")", + "got_raw": "\"q\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zappa/Zappa", + "expected": "\"async_me\")", + "got": "\"async_me\")", + "got_raw": "\"async_me\")", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zeenolife/ai-baby-monitor", + "expected": "mock_capture", + "got": "mock_capture", + "got_raw": "mock_capture", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zeenolife/ai-baby-monitor", + "expected": "approximate", + "got": "approximate", + "got_raw": "approximate", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zeenolife/ai-baby-monitor", + "expected": "1280", + "got": "1280", + "got_raw": "1280", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zeenolife/ai-baby-monitor", + "expected": "expected_logs_final_order", + "got": "expected_logs_final_order", + "got_raw": "expected_logs_final_order", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zeenolife/ai-baby-monitor", + "expected": "[]", + "got": "[]", + "got_raw": "[]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zerodha/pykiteconnect", + "expected": "kiteconnect.VARIETY_CO", + "got": "kiteconnect.VARIETY_CO", + "got_raw": "kiteconnect.VARIETY_CO", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zerodha/pykiteconnect", + "expected": "getattr(ex, error_type, ex.GeneralException))", + "got": "ex.NativeException)", + "got_raw": "ex.NativeException)", + "exact_match": false, + "code_bleu": 0.18477554478349628, + "edit_similarity": 0.4375 + }, + { + "repo": "zerodha/pykiteconnect", + "expected": "status", + "got": "status", + "got_raw": "status", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zerodha/pykiteconnect", + "expected": "20", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "zerodha/pykiteconnect", + "expected": "dict", + "got": "dict", + "got_raw": "dict", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "\"Sales by Region\"", + "got": "\"Sales by Region\"", + "got_raw": "\"Sales by Region\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "mock_instance1", + "got": "mock_instance1", + "got_raw": "mock_instance1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "\"llm_node\"", + "got": "\"llm_node\"", + "got_raw": "\"llm_node\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "\"English\"", + "got": "\"English\"", + "got_raw": "\"English\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "False", + "got": "True", + "got_raw": "True", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.2222222222222222 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "result", + "got": "result", + "got_raw": "result", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "True)", + "got": "knowledge_bases)", + "got_raw": "knowledge_bases)", + "exact_match": false, + "code_bleu": 0.537284965911771, + "edit_similarity": 0.19047619047619047 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "ValidationError)", + "got": "ValidationError)", + "got_raw": "ValidationError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "[\"test\"]", + "got": "[\"test\"]", + "got_raw": "[\"test\"]", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "20", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.5 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "description.lower()", + "got": "description.lower()", + "got_raw": "description.lower()", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "{}", + "got": "[]", + "got_raw": "[]", + "exact_match": false, + "code_bleu": 0.4518010018049224, + "edit_similarity": 0.0 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "\"call_2\"", + "got": "\"call_2\"", + "got_raw": "\"call_2\"", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "ids", + "got": "ids", + "got_raw": "ids", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "True", + "got": "True", + "got_raw": "True", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zhongyu09/openchatbi", + "expected": "TypeError)", + "got": "TypeError)", + "got_raw": "TypeError)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "s2.data)", + "got": "s2.data)", + "got_raw": "s2.data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "3", + "got": "2", + "got_raw": "2", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "[1, 3, 2])", + "got": "[1, 3, -6])", + "got_raw": "[1, 3, -6])", + "exact_match": false, + "code_bleu": 0.5814307369682193, + "edit_similarity": 0.8571428571428571 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "1", + "got": "1", + "got_raw": "1", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "b2", + "got": "b2", + "got_raw": "b2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "c.to_dict()", + "got": "c", + "got_raw": "c", + "exact_match": false, + "code_bleu": 0.11892071150027211, + "edit_similarity": 0.16666666666666666 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "c2", + "got": "c2", + "got_raw": "c2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "4", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "2", + "got": "2", + "got_raw": "2", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "errs", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "1", + "got": "0", + "got_raw": "0", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "8", + "got": "10", + "got_raw": "10", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "0", + "got": "0", + "got_raw": "0", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "0", + "got": "1", + "got_raw": "1", + "exact_match": false, + "code_bleu": 0.5, + "edit_similarity": 0.0 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "xs * ys)", + "got": "xs * ys)", + "got_raw": "xs * ys)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "8", + "got": "8", + "got_raw": "8", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + }, + { + "repo": "zwicker-group/py-pde", + "expected": "s2.data)", + "got": "s2.data)", + "got_raw": "s2.data)", + "exact_match": true, + "code_bleu": 1.0, + "edit_similarity": 1.0 + } + ] +} \ No newline at end of file